controllers

package
v0.0.0-...-417b102 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 27, 2023 License: Apache-2.0 Imports: 53 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	TUNED_MAX_QSIZE  = 100
	RANGE_MAX_LENGTH = 1000
	SET_MAX_LENGTH   = 20

	// opt params
	RANDOM_ROUND = 5
	MAX_ROUND    = 100

	CONFIG_FOLDER = "/etc/search-space"
)
View Source 
const (
	BENCHMARK_LABEL        = "cpe-benchmark"
	MAX_LABEL_LEN          = 60
	NODESELECT_ITR_NAME    = "profile"
	NODESELECT_ITR_DEFAULT = "default"
	INIT_BUILD_NAME        = "init"
	BUILD_KEY              = "build"
	REPETITION_KEY         = "repno"
	JOBHASH_KEY            = "cpe-jobhash"
	HASH_DELIMIT           = "-cpeh-"
	INVALID_REGEX          = "[^A-Za-z0-9]"
	JOB_LOG_PATH           = "/cpe-local-log"
)
View Source 
const (
	TUNED_RESOURCE      = "tuneds.v1.tuned.openshift.io"
	RENDERED_TUNED_NAME = "rendered"
	TUNED_NAMESPACE     = "openshift-cluster-node-tuning-operator"
	PROFILE_RESOURCE    = "profiles.v1.tuned.openshift.io"
	TUNED_KIND          = "Tuned"

	RESERVED_PRIORITY_NUMBER        = 0
	RESERVED_AUTOTUNED_PROFILE_NAME = "auto-tuned"
	BASE_PROFILE                    = "openshift-default"
)
View Source 
const BUILD_RESOURCE = "builds.v1.build.openshift.io"
View Source 
const JOB_MAX_QSIZE = 100
View Source 
const NO_BUILD_RESOURCE_ERROR = "the server could not find the requested resource"
View Source 
const ReconcileTime = 30 * time.Minute

Variables

View Source 
var CLUSTER_ID string = os.Getenv("CLUSTER_ID")
View Source 
var OperatorAdaptorMap map[string]OperatorAdaptor = map[string]OperatorAdaptor{
	"default":  defaultAdaptor,
	"ripsaw":   ripsawAdaptor,
	"mpi":      mpiAdaptor,
	"kubeflow": kubeflowAdaptor,
}
View Source 
var PARSE_RAW_URL string = os.Getenv("PARSER_SERVICE") + "/raw-parse"
View Source 
var PARSE_URL string = os.Getenv("PARSER_SERVICE") + "/parse"
View Source 
var PUSH_URL string = os.Getenv("PARSER_SERVICE") + "/push"
View Source 
var ParamNameMap map[string]TuneType
View Source 
var SearchSpace map[TuneType][]bo.Param

Functions

func CheckIfJobDone 

func CheckIfJobDone(benchmark *cpev1.Benchmark, jobName string) bool

func CreateFromOperator 

func CreateFromOperator(jtm *JobTrackManager, client client.Client, dc *discovery.DiscoveryClient, dyn dynamic.Interface, benchmark *cpev1.Benchmark, benchmarkOperator *cpev1.BenchmarkOperator, reqLogger logr.Logger, adaptor OperatorAdaptor, tunedHandler *TunedHandler) error

func CreateFromURL 

func CreateFromURL(dc *discovery.DiscoveryClient, dyn dynamic.Interface, yamlURL string) (*unstructured.Unstructured, error)

func CreateIfNotExists 

func CreateIfNotExists(dr dynamic.ResourceInterface, benchmark *cpev1.Benchmark, unstructuredInstance *unstructured.Unstructured, adaptor OperatorAdaptor, tunedHandler *TunedHandler, nodeTunedOptimizer *BaysesOptimizer) (error, bool)

func CreateIntUniformParam 

func CreateIntUniformParam(value string) (bo.Param, error)

func CreateSetParam 

func CreateSetParam(value string) (bo.Param, error)

func CreateUniformParam 

func CreateUniformParam(value string) (bo.Param, error)

func DeleteFromOperator 

func DeleteFromOperator(dc *discovery.DiscoveryClient, dyn dynamic.Interface, benchmark *cpev1.Benchmark, benchmarkOperator *cpev1.BenchmarkOperator) error

func DeleteFromURL 

func DeleteFromURL(dc *discovery.DiscoveryClient, dyn dynamic.Interface, yamlURL string) error

func Find 

func Find(slice []string, val string) (int, bool)

func GetAutoTunedProfile 

func GetAutoTunedProfile(tunedProfile map[TuneType]map[string]string) *unstructured.Unstructured

func GetBenchmarkWithIteration 

func GetBenchmarkWithIteration(client client.Client, ns string, benchmark *cpev1.Benchmark, benchmarkObj map[string]interface{}, iterationLabel map[string]string, build string, repetition int) (*unstructured.Unstructured, error)

func GetCombinedIterations 

func GetCombinedIterations(benchmark *cpev1.Benchmark) []cpev1.IterationItem

func GetDataProfile 

func GetDataProfile(tunedProfile map[TuneType]map[string]string) string

func GetDetailFromJobName 

func GetDetailFromJobName(jobName string, benchmark *cpev1.Benchmark) (benchmarkName string, iterationMap map[string]string, configurationMap map[string]string, repetition string, buildID string, iterationID string, configurationID string)

func GetExpandLabel 

func GetExpandLabel(iterationLabel map[string]string) map[string]interface{}

func GetInformerFromGVK 

func GetInformerFromGVK(dc *discovery.DiscoveryClient, dyn dynamic.Interface, gvk schema.GroupVersionKind) (cache.SharedIndexInformer, dynamicinformer.DynamicSharedInformerFactory)

func GetIteratedValues 

func GetIteratedValues(benchmark *cpev1.Benchmark) (firstLabel map[string]string, iterationLabels []map[string]string, builds []string, maxRepetition int)

func GetJobCompletedStatus 

func GetJobCompletedStatus(benchmark *cpev1.Benchmark) string

func GetSearchSpaceConfig 

func GetSearchSpaceConfig(configMapLoc string) (map[TuneType][]bo.Param, map[string]TuneType, error)

func GetSimpleJobGVK 

func GetSimpleJobGVK(benchmarkOperator *cpev1.BenchmarkOperator) schema.GroupVersionKind

func InitSearchSpace 

func InitSearchSpace() error

func JobListChanged 

func JobListChanged(benchmark *cpev1.Benchmark) bool

func NewBenchmarkObject 

func NewBenchmarkObject(benchmarkOperator *cpev1.BenchmarkOperator) map[string]interface{}

func NewCollector 

func NewCollector(client client.Client, logger logr.Logger)

func NodeSelectionSpecToIteration 

func NodeSelectionSpecToIteration(spec *cpev1.NodeSelectionSpec) cpev1.IterationItem

NodeSelectionSpec

Types

type BaseOperatorAdaptor 

type BaseOperatorAdaptor struct {
	OperatorAdaptor
}

Base Operartor Adaptor

type BaysesOptimizer 

type BaysesOptimizer struct {
	SampleQueue chan map[TuneType]map[string]string
	ResultQueue chan float64
	*bo.Optimizer
	Minimize              bool
	FinalizedTunedProfile map[TuneType]map[string]string
	AutoTuned             bool
	FinalizedReady        bool
	FinalizedApplied      bool
	SamplingCount         int
}

func NewBayesOptimizer 

func NewBayesOptimizer(minimize bool) *BaysesOptimizer

func (*BaysesOptimizer) AutoTune 

func (b *BaysesOptimizer) AutoTune()

func (*BaysesOptimizer) Finalize 

func (b *BaysesOptimizer) Finalize()

func (*BaysesOptimizer) SetFinalizedApplied 

func (b *BaysesOptimizer) SetFinalizedApplied()

type BenchmarkOperatorReconciler 

type BenchmarkOperatorReconciler struct {
	*kubernetes.Clientset
	client.Client
	Log        logr.Logger
	Scheme     *runtime.Scheme
	DC         *discovery.DiscoveryClient
	DYN        dynamic.Interface
	HelmClient helmclient.Client
}

BenchmarkOperatorReconciler reconciles a BenchmarkOperator object

func (*BenchmarkOperatorReconciler) DeployNoneOperator 

func (r *BenchmarkOperatorReconciler) DeployNoneOperator()

func (*BenchmarkOperatorReconciler) Reconcile 

func (r *BenchmarkOperatorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

func (*BenchmarkOperatorReconciler) SetupWithManager 

func (r *BenchmarkOperatorReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type BenchmarkReconciler 

type BenchmarkReconciler struct {
	client.Client
	Log    logr.Logger
	Scheme *runtime.Scheme
	DC     *discovery.DiscoveryClient
	DYN    dynamic.Interface
	JTM    *JobTrackManager
	*TunedHandler
}

BenchmarkReconciler reconciles a Benchmark object

func (*BenchmarkReconciler) Reconcile 

func (r *BenchmarkReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

func (*BenchmarkReconciler) SetupWithManager 

func (r *BenchmarkReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type BuildWatcher 

type BuildWatcher struct {
	client.Client
	Log        logr.Logger
	Scheme     *runtime.Scheme
	DC         *discovery.DiscoveryClient
	DYN        dynamic.Interface
	BuildQueue chan *unstructured.Unstructured
	Quit       chan struct{}
}

func (*BuildWatcher) InitInformer 

func (r *BuildWatcher) InitInformer() error

func (*BuildWatcher) ProcessBuildQueue 

func (r *BuildWatcher) ProcessBuildQueue()

func (*BuildWatcher) Run 

func (r *BuildWatcher) Run()

type COSObject 

type COSObject struct {
	RawBucketName     string
	APIKey            string
	ServiceInstanceID string
	AuthEndpoint      string
	ServiceEndpoint   string
}

func (*COSObject) InitValue 

func (c *COSObject) InitValue()

type DefaultAdaptor 

type DefaultAdaptor struct {
	*BaseOperatorAdaptor
}

Default Operartor Adaptor

func NewDefaultAdaptor 

func NewDefaultAdaptor() *DefaultAdaptor

func (*DefaultAdaptor) CheckComplete 

func (a *DefaultAdaptor) CheckComplete(jobObject map[string]interface{}) bool

func (*DefaultAdaptor) CopyJobResource 

func (a *DefaultAdaptor) CopyJobResource(originalJob *unstructured.Unstructured) *unstructured.Unstructured

func (*DefaultAdaptor) GetPodList 

func (a *DefaultAdaptor) GetPodList(jobObject map[string]interface{}, clientset *kubernetes.Clientset) (*corev1.PodList, error)

type IntUniformParam 

type IntUniformParam struct {
	Name           string
	Max, Min, Step int
}

func (IntUniformParam) GetMax 

func (p IntUniformParam) GetMax() float64

func (IntUniformParam) GetMin 

func (p IntUniformParam) GetMin() float64

func (IntUniformParam) GetName 

func (p IntUniformParam) GetName() string

func (IntUniformParam) Sample 

func (p IntUniformParam) Sample() float64

func (IntUniformParam) Validate 

func (p IntUniformParam) Validate(inValue float64) float64

type IterationHandler 

type IterationHandler struct{}

func (*IterationHandler) GetAllCombination 

func (it *IterationHandler) GetAllCombination(itr []cpev1.IterationItem) []map[string]string

func (*IterationHandler) GetValue 

func (it *IterationHandler) GetValue(initObject map[string]interface{}, location string) []string

func (*IterationHandler) UpdateValue 

func (it *IterationHandler) UpdateValue(baseObject map[string]interface{}, location string, value string) map[string]interface{}

type IterationKeyType 

type IterationKeyType int
const (
	VALUE IterationKeyType = iota
	LIST
	MAP
	NULL_VALUE_STR = "nil"
)

type JobTrackManager 

type JobTrackManager struct {
	client.Client
	*kubernetes.Clientset
	JobTrackers map[string]*JobTracker
	Cos         COSObject
	GlobalQuit  chan struct{}
	Log         logr.Logger
	DC          *discovery.DiscoveryClient
	DYN         dynamic.Interface
	*TunedHandler
}

func (*JobTrackManager) DeleteTracker 

func (m *JobTrackManager) DeleteTracker(jobGVK schema.GroupVersionKind, benchmarkName string)

func (*JobTrackManager) IsExist 

func (m *JobTrackManager) IsExist(jobGVK schema.GroupVersionKind, benchmarkName string) bool

func (*JobTrackManager) NewTracker 

func (m *JobTrackManager) NewTracker(jobGVK schema.GroupVersionKind, benchmarkName string, waitingJob []*unstructured.Unstructured, dr dynamic.ResourceInterface, adaptor OperatorAdaptor, jobOptMap map[string]*BaysesOptimizer)

func (*JobTrackManager) Run 

func (m *JobTrackManager) Run()

type JobTracker 

type JobTracker struct {
	client.Client
	*kubernetes.Clientset
	Log            logr.Logger
	DC             *discovery.DiscoveryClient
	DYN            dynamic.Interface
	JobQueue       chan *unstructured.Unstructured
	Quit           chan struct{}
	JobGVK         schema.GroupVersionKind
	Cos            COSObject
	Subscribers    []string
	WaitingJobMap  map[string][]*unstructured.Unstructured
	DRMap          map[string]dynamic.ResourceInterface
	Adaptor        OperatorAdaptor
	JobOptMap      map[string]*BaysesOptimizer
	BestPodNameMap map[string]string
	*TunedHandler
}

func (*JobTracker) End 

func (r *JobTracker) End()

func (*JobTracker) Init 

func (r *JobTracker) Init()

func (*JobTracker) IsEmpty 

func (r *JobTracker) IsEmpty() bool

func (*JobTracker) IsExist 

func (r *JobTracker) IsExist(benchmarkName string) bool

func (*JobTracker) ProcessJobQueue 

func (r *JobTracker) ProcessJobQueue()

func (*JobTracker) Run 

func (r *JobTracker) Run()

func (*JobTracker) Subscribe 

func (r *JobTracker) Subscribe(benchmarkName string, waitingJob []*unstructured.Unstructured, dr dynamic.ResourceInterface, jobOptMap map[string]*BaysesOptimizer)

func (*JobTracker) Unsubscribe 

func (r *JobTracker) Unsubscribe(benchmarkName string)

type KubeflowAdaptor 

type KubeflowAdaptor struct {
	*BaseOperatorAdaptor
}

Kubeflow Operartor Adaptor

func NewKubeflowAdaptor 

func NewKubeflowAdaptor() *KubeflowAdaptor

func (*KubeflowAdaptor) CheckComplete 

func (a *KubeflowAdaptor) CheckComplete(jobObject map[string]interface{}) bool

func (*KubeflowAdaptor) CopyJobResource 

func (a *KubeflowAdaptor) CopyJobResource(originalJob *unstructured.Unstructured) *unstructured.Unstructured

func (*KubeflowAdaptor) GetPodList 

func (a *KubeflowAdaptor) GetPodList(jobObject map[string]interface{}, clientset *kubernetes.Clientset) (*corev1.PodList, error)

type LogSpec 

type LogSpec struct {
	ClusterID     string            `json:"cluster"`
	Instance      string            `json:"instance"`
	BenchmarkName string            `json:"benchmark"`
	JobName       string            `json:"job"`
	PodName       string            `json:"pod"`
	Parser        string            `json:"parser"`
	ConstLabels   map[string]string `json:"labels"`
}

type MPIAdaptor 

type MPIAdaptor struct {
	*BaseOperatorAdaptor
}

MPI Operartor Adaptor

func NewMPIAdaptor 

func NewMPIAdaptor() *MPIAdaptor

func (*MPIAdaptor) CheckComplete 

func (a *MPIAdaptor) CheckComplete(jobObject map[string]interface{}) bool

func (*MPIAdaptor) CopyJobResource 

func (a *MPIAdaptor) CopyJobResource(originalJob *unstructured.Unstructured) *unstructured.Unstructured

func (*MPIAdaptor) GetPodList 

func (a *MPIAdaptor) GetPodList(jobObject map[string]interface{}, clientset *kubernetes.Clientset) (*corev1.PodList, error)

type OperatorAdaptor 

type OperatorAdaptor interface {
	CheckComplete(jobObject map[string]interface{}) bool
	GetPodList(jobObject map[string]interface{}, clientset *kubernetes.Clientset) (*corev1.PodList, error)
	CopyJobResource(originalJob *unstructured.Unstructured) *unstructured.Unstructured
}

type ParamType 

type ParamType string

func (ParamType) GetParam 

func (t ParamType) GetParam(fileName string, tuneType TuneType, paramNameMap map[string]TuneType) ([]bo.Param, error)

type RawLog 

type RawLog struct {
	Parser   string `json:"parser"`
	LogValue []byte `json:"log"`
}

type Response 

type Response struct {
	Status           string  `json:"status"`
	Message          string  `json:"msg"`
	PerformanceKey   string  `json:"pkey"`
	PerformanceValue float64 `json:"pval"`
}

type ResultCollector 

type ResultCollector struct {
	client.Client
	Log logr.Logger
	// contains filtered or unexported fields
}

func (*ResultCollector) Collect 

func (c *ResultCollector) Collect(ch chan<- prometheus.Metric)

Collect implements the prometheus.Collector interface "benchmark", "build", "configID", "scenarioID", "job", "pod", "key", "index"

func (*ResultCollector) Describe 

func (c *ResultCollector) Describe(ch chan<- *prometheus.Desc)

Describe implements the prometheus.Collector interface

type RipsawAdaptor 

type RipsawAdaptor struct {
	*BaseOperatorAdaptor
}

Ripsaw Operartor Adaptor

func NewRipsawAdaptor 

func NewRipsawAdaptor() *RipsawAdaptor

func (*RipsawAdaptor) CheckComplete 

func (a *RipsawAdaptor) CheckComplete(jobObject map[string]interface{}) bool

func (*RipsawAdaptor) CopyJobResource 

func (a *RipsawAdaptor) CopyJobResource(originalJob *unstructured.Unstructured) *unstructured.Unstructured

func (*RipsawAdaptor) GetPodList 

func (a *RipsawAdaptor) GetPodList(jobObject map[string]interface{}, clientset *kubernetes.Clientset) (*corev1.PodList, error)

type SetParam 

type SetParam struct {
	Name      string
	Values    [SET_MAX_LENGTH]string
	SetLength int
}

func (SetParam) GetMax 

func (p SetParam) GetMax() float64

func (SetParam) GetMin 

func (p SetParam) GetMin() float64

func (SetParam) GetName 

func (p SetParam) GetName() string

func (SetParam) GetSetValue 

func (p SetParam) GetSetValue(index float64) string

func (SetParam) Sample 

func (p SetParam) Sample() float64

func (SetParam) Validate 

func (p SetParam) Validate(value float64) float64

type TuneType 

type TuneType string

func (TuneType) IsValid 

func (t TuneType) IsValid() error

type TunedHandler 

type TunedHandler struct {
	*kubernetes.Clientset
	Log logr.Logger
	DYN dynamic.Interface
}

func (*TunedHandler) ApplyProfile 

func (t *TunedHandler) ApplyProfile(nodeSelector *metav1.LabelSelector, profileName string)

func (*TunedHandler) CreateAutoTunedProfile 

func (t *TunedHandler) CreateAutoTunedProfile(tunedProfile map[TuneType]map[string]string) error

func (*TunedHandler) DeleteAutoTunedProfile 

func (t *TunedHandler) DeleteAutoTunedProfile() error

func (*TunedHandler) DeleteLabel 

func (t *TunedHandler) DeleteLabel(nodeSelector *metav1.LabelSelector)

func (*TunedHandler) IsSameProfile 

func (t *TunedHandler) IsSameProfile(tunedProfile map[TuneType]map[string]string, cmpTunedProfile map[TuneType]map[string]string) bool

type ValueWithLabels 

type ValueWithLabels struct {
	Labels map[string]string
	Value  float64
}

type ValuesWithLabels 

type ValuesWithLabels struct {
	Labels map[string]string
	Values []float64
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.