controllers

package
v1.6.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 5, 2024 License: Apache-2.0 Imports: 40 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	DBAvailableMetric = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "data_science_pipelines_application_database_available",
			Help: "Data Science Pipelines Application - Database Availability Status",
		},
		[]string{
			"dspa_name",
			"dspa_namespace",
		},
	)
	ObjectStoreAvailableMetric = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "data_science_pipelines_application_object_store_available",
			Help: "Data Science Pipelines Application - Object Store Availability Status",
		},
		[]string{
			"dspa_name",
			"dspa_namespace",
		},
	)
	APIServerReadyMetric = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "data_science_pipelines_application_apiserver_ready",
			Help: "Data Science Pipelines Application - APIServer Ready Status",
		},
		[]string{
			"dspa_name",
			"dspa_namespace",
		},
	)
	PersistenceAgentReadyMetric = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "data_science_pipelines_application_persistenceagent_ready",
			Help: "Data Science Pipelines Application - PersistenceAgent Ready Status",
		},
		[]string{
			"dspa_name",
			"dspa_namespace",
		},
	)
	ScheduledWorkflowReadyMetric = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "data_science_pipelines_application_scheduledworkflow_ready",
			Help: "Data Science Pipelines Application - ScheduledWorkflow Ready Status",
		},
		[]string{
			"dspa_name",
			"dspa_namespace",
		},
	)
	CrReadyMetric = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "data_science_pipelines_application_ready",
			Help: "Data Science Pipelines Application - CustomResource Ready Status",
		},
		[]string{
			"dspa_name",
			"dspa_namespace",
		},
	)
)

Prometheus metrics gauges

View Source 
var ConnectAndQueryDatabase = func(
	host string,
	log logr.Logger,
	port, username, password, dbname, tls string,
	pemCerts [][]byte,
	extraParams map[string]string) (bool, error) {

	mysqlConfig := createMySQLConfig(
		username,
		password,
		host,
		port,
		"",
		extraParams,
	)

	ctx, cancel := context.WithTimeout(context.Background(), config.DefaultDBConnectionTimeout)
	defer cancel()

	var tlsConfig *cryptoTls.Config
	switch tls {
	case "false", "":

	case "true":
		if len(pemCerts) != 0 {
			var err error
			tlsConfig, err = tLSClientConfig(pemCerts)
			if err != nil {
				log.Info(fmt.Sprintf("Encountered error when processing custom ca bundle, Error: %v", err))
				return false, err
			}
		}
	case "skip-verify", "preferred":
		tlsConfig = &cryptoTls.Config{InsecureSkipVerify: true}
	default:

	}

	if tlsConfig != nil {
		err := mysql.RegisterTLSConfig("custom", tlsConfig)

		if _, ok := mysqlConfig.Params["tls"]; ok {
			mysqlConfig.Params["tls"] = "custom"
		}

		mysqlConfig.TLSConfig = "custom"
		if err != nil {
			return false, err
		}
	}

	db, err := sql.Open("mysql", mysqlConfig.FormatDSN())
	if err != nil {
		return false, err
	}
	defer db.Close()

	testStatement := "SELECT 1;"
	_, err = db.QueryContext(ctx, testStatement)
	if err != nil {
		return false, err
	}
	return true, nil
}
View Source 
var ConnectAndQueryObjStore = func(
	ctx context.Context,
	log logr.Logger,
	endpoint, bucket string,
	accesskey, secretkey []byte,
	secure bool,
	pemCerts [][]byte) bool {
	cred := createCredentialProvidersChain(string(accesskey), string(secretkey))

	opts := &minio.Options{
		Creds:  cred,
		Secure: secure,
	}

	if len(pemCerts) != 0 {
		tr, err := getHttpsTransportWithCACert(log, pemCerts)
		if err != nil {
			log.Error(err, "Encountered error when processing custom ca bundle.")
			return false
		}
		opts.Transport = tr
	}

	minioClient, err := minio.New(endpoint, opts)
	if err != nil {
		log.Info(fmt.Sprintf("Could not connect to object storage endpoint: %s", endpoint))
		return false
	}

	ctx, cancel := context.WithTimeout(ctx, config.DefaultObjStoreConnectionTimeout)
	defer cancel()

	_, err = minioClient.StatObject(ctx, bucket, "some-random-object", minio.GetObjectOptions{})
	if err != nil {
		switch err := err.(type) {

		case minio.ErrorResponse:
			if err.Code == "NoSuchKey" || err.Code == "NoSuchBucket" {
				return true
			}
		}

		if util.IsX509UnknownAuthorityError(err) {
			log.Info(fmt.Sprintf("Encountered x509 UnknownAuthorityError when connecting to ObjectStore. "+
				"If using an tls S3 connection with  self-signed certs, you may specify a custom CABundle "+
				"to mount on the DSP API Server via the DSPA cr under the spec.cABundle field. If you have already "+
				"provided a CABundle, verify the validity of the provided CABundle. Error: %v", err))
			return false
		}

		log.Info(fmt.Sprintf("Could not connect to (%s), Error: %s", endpoint, err.Error()))
		return false
	}

	return true
}

Functions

func InitMetrics added in v0.2.0 

func InitMetrics()

InitMetrics initialize prometheus metrics

Types

type DBConnection 

type DBConnection struct {
	Host              string
	Port              string
	Username          string
	DBName            string
	CredentialsSecret *dspa.SecretKeyValue
	Password          string
	ExtraParams       string
}

type DSPAParams 

type DSPAParams struct {
	Name                                 string
	Namespace                            string
	Owner                                mf.Owner
	APIServer                            *dspa.APIServer
	APIServerDefaultResourceName         string
	APIServerServiceName                 string
	OAuthProxy                           string
	ScheduledWorkflow                    *dspa.ScheduledWorkflow
	ScheduledWorkflowDefaultResourceName string
	PersistenceAgent                     *dspa.PersistenceAgent
	PersistentAgentDefaultResourceName   string
	MlPipelineUI                         *dspa.MlPipelineUI
	MariaDB                              *dspa.MariaDB
	Minio                                *dspa.Minio
	MLMD                                 *dspa.MLMD
	DBConnection
	ObjectStorageConnection

	// TLS
	// The CA bundle path used by API server
	CustomCABundleRootMountPath string
	// This path is used by API server to also look
	// for CustomCABundleRootMountPath when
	// verifying certs
	CustomSSLCertDir *string
	// The CA bundle path found in the pipeline pods
	PiplinesCABundleMountPath string
	// Collects all certs from user & global certs
	APICustomPemCerts [][]byte
	// Source of truth for the DSP cert configmap details
	// If this is defined, then we assume we have additional certs
	// we need to leverage for tls connections within dsp apiserver
	// pipeline pods
	CustomCABundle *dspa.CABundle
}

func (*DSPAParams) DatabaseHealthCheckDisabled added in v1.2.0 

func (p *DSPAParams) DatabaseHealthCheckDisabled(dsp *dspa.DataSciencePipelinesApplication) bool

DatabaseHealthCheckDisabled will return the value if the Database has disableHealthCheck specified in the CR, otherwise false.

func (*DSPAParams) ExtractParams 

func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, loggr logr.Logger) error

func (*DSPAParams) ObjectStorageHealthCheckDisabled added in v1.2.0 

func (p *DSPAParams) ObjectStorageHealthCheckDisabled(dsp *dspa.DataSciencePipelinesApplication) bool

ObjectStorageHealthCheckDisabled will return the value if the Object Storage has disableHealthCheck specified in the CR, otherwise false.

func (*DSPAParams) SetupDBParams 

func (p *DSPAParams) SetupDBParams(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, log logr.Logger) error

SetupDBParams Populates the DB connection Parameters. If an external secret is specified, SetupDBParams will retrieve DB credentials from it. If DSPO is managing a dynamically created secret, then SetupDBParams generates the creds.

func (*DSPAParams) SetupMLMD added in v0.2.2 

func (p *DSPAParams) SetupMLMD(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, log logr.Logger) error

func (*DSPAParams) SetupObjectParams 

func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, log logr.Logger) error

SetupObjectParams Populates the Object Storage connection Parameters. If an external secret is specified, SetupObjectParams will retrieve storage credentials from it. If DSPO is managing a dynamically created secret, then SetupObjectParams generates the creds.

func (*DSPAParams) UsingExternalDB 

func (p *DSPAParams) UsingExternalDB(dsp *dspa.DataSciencePipelinesApplication) bool

UsingExternalDB will return true if an external Database is specified in the CR, otherwise false.

func (*DSPAParams) UsingExternalStorage 

func (p *DSPAParams) UsingExternalStorage(dsp *dspa.DataSciencePipelinesApplication) bool

UsingExternalStorage will return true if an external Object Storage is specified in the CR, otherwise false.

func (*DSPAParams) UsingMLMD added in v0.2.2 

func (p *DSPAParams) UsingMLMD(dsp *dspa.DataSciencePipelinesApplication) bool

type DSPAReconciler 

type DSPAReconciler struct {
	client.Client
	Scheme                  *runtime.Scheme
	Log                     logr.Logger
	TemplatesPath           string
	MaxConcurrentReconciles int
}

DSPAReconciler reconciles a DSPAParams object

func (*DSPAReconciler) Apply 

func (r *DSPAReconciler) Apply(owner mf.Owner, params *DSPAParams, template string, fns ...mf.Transformer) error

func (*DSPAReconciler) ApplyWithoutOwner 

func (r *DSPAReconciler) ApplyWithoutOwner(params *DSPAParams, template string, fns ...mf.Transformer) error

func (*DSPAReconciler) CleanUpCommon 

func (r *DSPAReconciler) CleanUpCommon(params *DSPAParams) error

func (*DSPAReconciler) DeleteResource 

func (r *DSPAReconciler) DeleteResource(params *DSPAParams, template string, fns ...mf.Transformer) error

func (*DSPAReconciler) DeleteResourceIfItExists 

func (r *DSPAReconciler) DeleteResourceIfItExists(ctx context.Context, obj client.Object, nn types.NamespacedName) error

func (*DSPAReconciler) GenerateStatus added in v0.2.2 

func (r *DSPAReconciler) GenerateStatus(ctx context.Context, dspa *dspav1alpha1.DataSciencePipelinesApplication,
	params *DSPAParams, dbAvailableStatus, objStoreAvailableStatus bool) ([]metav1.Condition, error)

func (*DSPAReconciler) PublishMetrics added in v0.2.0 

func (r *DSPAReconciler) PublishMetrics(dspa *dspav1alpha1.DataSciencePipelinesApplication, metricsMap map[metav1.Condition]*prometheus.GaugeVec)

func (*DSPAReconciler) Reconcile 

func (r *DSPAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

func (*DSPAReconciler) ReconcileAPIServer 

func (r *DSPAReconciler) ReconcileAPIServer(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication, params *DSPAParams) error

func (*DSPAReconciler) ReconcileCommon 

func (r *DSPAReconciler) ReconcileCommon(dsp *dspav1alpha1.DataSciencePipelinesApplication, params *DSPAParams) error

func (*DSPAReconciler) ReconcileDatabase 

func (r *DSPAReconciler) ReconcileDatabase(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication,
	params *DSPAParams) error

func (*DSPAReconciler) ReconcileMLMD added in v0.2.2 

func (r *DSPAReconciler) ReconcileMLMD(dsp *dspav1alpha1.DataSciencePipelinesApplication,
	params *DSPAParams) error

func (*DSPAReconciler) ReconcilePersistenceAgent 

func (r *DSPAReconciler) ReconcilePersistenceAgent(dsp *dspav1alpha1.DataSciencePipelinesApplication,
	params *DSPAParams) error

func (*DSPAReconciler) ReconcileScheduledWorkflow 

func (r *DSPAReconciler) ReconcileScheduledWorkflow(dsp *dspav1alpha1.DataSciencePipelinesApplication,
	params *DSPAParams) error

func (*DSPAReconciler) ReconcileStorage 

func (r *DSPAReconciler) ReconcileStorage(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication,
	params *DSPAParams) error

ReconcileStorage will set up Storage Connection.

func (*DSPAReconciler) ReconcileUI 

func (r *DSPAReconciler) ReconcileUI(dsp *dspav1alpha1.DataSciencePipelinesApplication,
	params *DSPAParams) error

func (*DSPAReconciler) SetupWithManager 

func (r *DSPAReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type ObjectStorageConnection 

type ObjectStorageConnection struct {
	Bucket            string
	CredentialsSecret *dspa.S3CredentialSecret
	Host              string
	Port              string
	Scheme            string
	Secure            *bool
	Endpoint          string // scheme://host:port
	AccessKeyID       string
	SecretAccessKey   string
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.