manager

package
v0.0.0-...-6005cdf Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 28, 2021 License: Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// MonitorDataSourceIntervalSeconds is interval time of monitoring data source
	MonitorDataSourceIntervalSeconds = 10
	// DatasetResourceKind is kind of dataset resource
	DatasetResourceKind = "dataset"
)
View Source 
const (
	// JobIterationIntervalSeconds is interval time of each iteration of job
	JobIterationIntervalSeconds = 10
	// DatasetHandlerIntervalSeconds is interval time of handling dataset
	DatasetHandlerIntervalSeconds = 10
	// ModelHandlerIntervalSeconds is interval time of handling model
	ModelHandlerIntervalSeconds = 10
	// EvalSamplesCapacity is capacity of eval samples
	EvalSamplesCapacity = 5
	//IncrementalLearningJobKind is kind of incremental-learning-job resource
	IncrementalLearningJobKind = "incrementallearningjob"
)
View Source 
const (
	// ModelCacheSize is size of cache
	ModelCacheSize = 100
	// ModelResourceKind is kind of dataset resource
	ModelResourceKind = "model"
)
View Source 
const (
	// WorkerMessageChannelCacheSize is size of channel cache
	WorkerMessageChannelCacheSize = 100

	// TrainPhase is the train phase in incremental-learning-job
	TrainPhase = "train"
	// EvalPhase is the eval phase in incremental-learning-job
	EvalPhase = "eval"
	// DeployPhase is the deploy phase in incremental-learning-job
	DeployPhase = "deploy"

	// WorkerWaitingStatus is the waiting status about worker
	WorkerWaitingStatus = "waiting"

	// WorkerReadyStatus is the ready status about worker
	WorkerReadyStatus = "ready"
	// WorkerCompletedStatus is the completed status about worker
	WorkerCompletedStatus = "completed"
	// WorkerFailedStatus is the failed status about worker
	WorkerFailedStatus = "failed"

	// TriggerReadyStatus is the ready status about trigger in incremental-learning-job
	TriggerReadyStatus = "ready"
	// TriggerCompletedStatus is the completed status about trigger in incremental-learning-job
	TriggerCompletedStatus = "completed"
)
View Source 
const (
	//FederatedLearningJobKind is kind of federated-learning-job resource
	FederatedLearningJobKind = "federatedlearningjob"
)
View Source 
const (
	// JointInferenceServiceKind is kind of joint-inference-service resource
	JointInferenceServiceKind = "jointinferenceservice"
)

Variables

This section is empty.

Functions

This section is empty.

Types

type DataSamples 

type DataSamples struct {
	Numbers            int
	TrainSamples       []string
	EvalVersionSamples [][]string
	EvalSamples        []string
}

DataSamples defines samples information

type DataSource 

type DataSource struct {
	TrainSamples    []string `json:"trainSamples"`
	ValidSamples    []string `json:"validSamples"`
	NumberOfSamples int      `json:"numberOfSamples"`
}

DataSource defines config for data source

type Dataset 

type Dataset struct {
	*neptunev1.Dataset
	DataSource *DataSource `json:"dataSource"`
	Done       chan struct{}
}

Dataset defines config for dataset

type DatasetManager 

type DatasetManager struct {
	Client            gmclient.ClientI
	DatasetMap        map[string]*Dataset
	VolumeMountPrefix string
}

DatasetManager defines dataset manager

func NewDatasetManager 

func NewDatasetManager(client gmclient.ClientI, options *options.LocalControllerOptions) *DatasetManager

NewDatasetManager creates a dataset manager

func (*DatasetManager) AddWorkerMessage 

func (dm *DatasetManager) AddWorkerMessage(message WorkerMessage)

func (*DatasetManager) Delete 

func (dm *DatasetManager) Delete(message *gmclient.Message) error

Delete deletes dataset config in db

func (*DatasetManager) GetDataset 

func (dm *DatasetManager) GetDataset(name string) (*Dataset, bool)

GetDatasetChannel gets dataset

func (*DatasetManager) GetName 

func (dm *DatasetManager) GetName() string

func (*DatasetManager) Insert 

func (dm *DatasetManager) Insert(message *gmclient.Message) error

Insert inserts dataset to db

func (*DatasetManager) Start 

func (dm *DatasetManager) Start() error

Start starts dataset manager

type DatasetSpec 

type DatasetSpec struct {
	Format  string `json:"format"`
	DataURL string `json:"url"`
}

DatasetSpec defines dataset spec

type FeatureManager 

type FeatureManager interface {
	// Start starts the manager
	Start() error

	// GetName returns name of the manager
	GetName() string

	// AddWorkerMessage dispatch the worker message to manager
	AddWorkerMessage(message WorkerMessage)

	// Insert includes gm message creation/updation
	Insert(*gmclient.Message) error

	Delete(*gmclient.Message) error
}

FeatureManager defines feature manager

func NewFederatedLearningManager 

func NewFederatedLearningManager(client gmclient.ClientI) FeatureManager

NewFederatedLearningManager creates a federated-learning-job manager

func NewJointInferenceManager 

func NewJointInferenceManager(client gmclient.ClientI) FeatureManager

NewJointInferenceManager creates a joint inference manager

type FederatedLearning 

type FederatedLearning struct {
	*neptunev1.FederatedLearningJob
}

FederatedLearning defines config for federated-learning-job

type FederatedLearningManager 

type FederatedLearningManager struct {
	Client               gmclient.ClientI
	WorkerMessageChannel chan WorkerMessage
}

FederatedLearningManager defines federated-learning-job manager

func (*FederatedLearningManager) AddWorkerMessage 

func (fm *FederatedLearningManager) AddWorkerMessage(message WorkerMessage)

AddWorkerMessage adds worker messages to the channel

func (*FederatedLearningManager) Delete 

func (fm *FederatedLearningManager) Delete(message *gmclient.Message) error

Delete deletes federated-learning-job config in db

func (*FederatedLearningManager) GetName 

func (fm *FederatedLearningManager) GetName() string

GetName returns the name of the manager

func (*FederatedLearningManager) Insert 

func (fm *FederatedLearningManager) Insert(message *gmclient.Message) error

Insert inserts federated-learning-job config in db

func (*FederatedLearningManager) Start 

func (fm *FederatedLearningManager) Start() error

Start starts federated-learning-job manager

type IncrementalJobManager 

type IncrementalJobManager struct {
	Client               gmclient.ClientI
	WorkerMessageChannel chan WorkerMessage
	DatasetManager       *DatasetManager
	ModelManager         *ModelManager
	IncrementalJobMap    map[string]*IncrementalLearningJob
	VolumeMountPrefix    string
}

IncrementalLearningJob defines incremental-learning-job manager

func NewIncrementalJobManager 

func NewIncrementalJobManager(client gmclient.ClientI, datasetManager *DatasetManager,
	modelManager *ModelManager, options *options.LocalControllerOptions) *IncrementalJobManager

NewIncrementalJobManager creates a incremental-learning-job manager

func (*IncrementalJobManager) AddWorkerMessage 

func (im *IncrementalJobManager) AddWorkerMessage(message WorkerMessage)

AddWorkerMessage adds worker messages

func (*IncrementalJobManager) Delete 

func (im *IncrementalJobManager) Delete(message *gmclient.Message) error

Delete deletes incremental-learning-job config in db

func (*IncrementalJobManager) GetName 

func (im *IncrementalJobManager) GetName() string

GetName returns name of the manager

func (*IncrementalJobManager) Insert 

func (im *IncrementalJobManager) Insert(message *gmclient.Message) error

Insert inserts incremental-learning-job config to db

func (*IncrementalJobManager) Start 

func (im *IncrementalJobManager) Start() error

Start starts incremental-learning-job manager

type IncrementalLearningJob 

type IncrementalLearningJob struct {
	neptunev1.IncrementalLearningJob
	JobConfig *JobConfig
	Dataset   *Dataset
	Done      chan struct{}
}

IncrementalLearningJob defines config for incremental-learning-job

type JobConfig 

type JobConfig struct {
	UniqueIdentifier string
	Version          int
	Phase            string
	WorkerStatus     string
	TrainTrigger     trigger.Base
	DeployTrigger    trigger.Base
	TriggerStatus    string
	TriggerTime      time.Time
	TrainDataURL     string
	EvalDataURL      string
	OutputDir        string
	OutputConfig     *OutputConfig
	DataSamples      *DataSamples
	TrainModel       *TrainModel
	DeployModel      *ModelInfo
	EvalResult       []*ModelInfo
	Lock             sync.Mutex
}

JobConfig defines config for incremental-learning-job

type JointInferenceManager 

type JointInferenceManager struct {
	Client               gmclient.ClientI
	WorkerMessageChannel chan WorkerMessage
}

JointInferenceManager defines joint-inference-service manager

func (*JointInferenceManager) AddWorkerMessage 

func (jm *JointInferenceManager) AddWorkerMessage(message WorkerMessage)

AddWorkerMessage adds worker messages

func (*JointInferenceManager) Delete 

func (jm *JointInferenceManager) Delete(message *gmclient.Message) error

Delete deletes joint-inference-service config in db

func (*JointInferenceManager) GetName 

func (jm *JointInferenceManager) GetName() string

GetName gets kind of the manager

func (*JointInferenceManager) Insert 

func (jm *JointInferenceManager) Insert(message *gmclient.Message) error

Insert inserts joint-inference-service config in db

func (*JointInferenceManager) Start 

func (jm *JointInferenceManager) Start() error

Start starts joint-inference-service manager

type MetaData 

type MetaData struct {
	Name      string `json:"name"`
	Namespace string `json:"namespace"`
}

MetaData defines metadata

type ModelInfo 

type ModelInfo struct {
	Format  string               `json:"format"`
	URL     string               `json:"url"`
	Metrics map[string][]float64 `json:"metrics,omitempty"`
}

ModelInfo defines model

type ModelManager 

type ModelManager struct {
	Client   gmclient.ClientI
	ModelMap map[string]neptunev1.Model
}

ModelManager defines model manager

func NewModelManager 

func NewModelManager(client gmclient.ClientI) *ModelManager

NewModelManager creates a model manager

func (*ModelManager) AddWorkerMessage 

func (mm *ModelManager) AddWorkerMessage(message WorkerMessage)

func (*ModelManager) Delete 

func (mm *ModelManager) Delete(message *gmclient.Message) error

Delete deletes model in db

func (*ModelManager) GetModel 

func (mm *ModelManager) GetModel(name string) (neptunev1.Model, bool)

GetModel gets model

func (*ModelManager) GetName 

func (mm *ModelManager) GetName() string

func (*ModelManager) Insert 

func (mm *ModelManager) Insert(message *gmclient.Message) error

insertModel inserts model config to db

func (*ModelManager) Start 

func (mm *ModelManager) Start() error

Start starts model manager

type OutputConfig 

type OutputConfig struct {
	SamplesOutput map[string]string `json:"trainData"`
	TrainOutput   string            `json:"trainOutput"`
	EvalOutput    string            `json:"evalOutput"`
}

OutputConfig defines config for job output

type TrainModel 

type TrainModel struct {
	Model        *ModelInfo        `json:"model"`
	TrainedModel map[string]string `json:"trainedModel"`
	OutputURL    string            `json:"outputUrl"`
}

TrainModel defines config about training model

type UpstreamMessage 

type UpstreamMessage struct {
	Phase  string        `json:"phase"`
	Status string        `json:"status"`
	Input  *WorkerInput  `json:"input,omitempty"`
	Output *WorkerOutput `json:"output"`
}

UpstreamMessage defines send message to GlobalManager

type WorkerInput 

type WorkerInput struct {
	// Only one model cases
	Models []ModelInfo `json:"models,omitempty"`

	DataURL   string `json:"dataURL,omitempty"`
	OutputDir string `json:"outputDir,omitempty"`
}

type WorkerMessage 

type WorkerMessage struct {
	Name      string                   `json:"name"`
	Namespace string                   `json:"namespace"`
	OwnerName string                   `json:"ownerName"`
	OwnerKind string                   `json:"ownerKind"`
	Kind      string                   `json:"kind"`
	Status    string                   `json:"status"`
	OwnerInfo map[string]interface{}   `json:"ownerInfo"`
	Results   []map[string]interface{} `json:"results"`
}

WorkerMessage defines message struct from worker

type WorkerOutput 

type WorkerOutput struct {
	Models    []map[string]interface{} `json:"models"`
	OwnerInfo map[string]interface{}   `json:"ownerInfo"`
}

WorkerOutput defines output information of worker

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.