entity

package
v0.0.0-...-a23f117 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 9, 2023 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type ComponentAlgorithmType

type ComponentAlgorithmType uint8

ComponentAlgorithmType is the type enum of the algorithm

const (
	ComponentAlgorithmTypeUnknown ComponentAlgorithmType = iota
	ComponentAlgorithmTypeHomoLR
	ComponentAlgorithmTypeHomoSBT
)

type FATEFlowContext

type FATEFlowContext struct {
	// FATEFlowHost is the host address of the service
	FATEFlowHost string
	// FATEFlowPort is the port of the service
	FATEFlowPort uint
	// FATEFlowIsHttps is whether the connection should be over TLS
	FATEFlowIsHttps bool
}

FATEFlowContext currently contains FATE flow connection info

type Headers

type Headers []string

Headers are local data column names

func (*Headers) Scan

func (h *Headers) Scan(v interface{}) error

func (Headers) Value

func (h Headers) Value() (driver.Value, error)

type Job

type Job struct {
	gorm.Model
	Name                   string    `json:"name" gorm:"type:varchar(255)"`
	Description            string    `json:"description" gorm:"type:text"`
	UUID                   string    `json:"uuid" gorm:"type:varchar(36)"`
	ProjectUUID            string    `json:"project_uuid" gorm:"type:varchar(36)"`
	Type                   JobType   `json:"type"`
	Status                 JobStatus `json:"status"`
	StatusMessage          string    `gorm:"type:text"`
	AlgorithmType          JobAlgorithmType
	AlgorithmComponentName string                      `json:"algorithm_component_name" gorm:"type:varchar(255)"`
	EvaluateComponentName  string                      `json:"evaluate_component_name" gorm:"type:varchar(255)"`
	AlgorithmConfig        valueobject.AlgorithmConfig `gorm:"type:text"`
	ModelName              string                      `json:"model_name" gorm:"type:varchar(255)"`
	PredictingModelUUID    string                      `gorm:"type:varchar(36)"`
	InitiatingSiteUUID     string                      `gorm:"type:varchar(36)"`
	InitiatingSiteName     string                      `gorm:"type:varchar(255)"`
	InitiatingSitePartyID  uint
	InitiatingUser         string `gorm:"type:varchar(255)"`
	IsInitiatingSite       bool
	FATEJobID              string `gorm:"type:varchar(255);column:fate_job_id"`
	FATEJobStatus          string `gorm:"type:varchar(36);column:fate_job_status"`
	FATEModelID            string `gorm:"type:varchar(255);column:fate_model_id"`
	FATEModelVersion       string `gorm:"type:varchar(255);column:fate_model_version"`
	FinishedAt             time.Time
	ResultJson             string             `gorm:"type:text"`
	Conf                   string             `gorm:"type:text"`
	DSL                    string             `gorm:"type:text"`
	RequestJson            string             `gorm:"type:text"`
	FATEFlowContext        FATEFlowContext    `gorm:"-"`
	Repo                   repo.JobRepository `gorm:"-"`
}

Job represents a FATE job

func (*Job) CheckFATEJobStatus

func (job *Job) CheckFATEJobStatus() error

CheckFATEJobStatus issues job status query

func (*Job) Create

func (job *Job) Create() error

Create initializes the job and save into the repo. The uuid will be automatically generated if not set

func (*Job) GetIntersectionResult

func (job *Job) GetIntersectionResult() ([]string, [][]interface{}, int, float64)

GetIntersectionResult parses the PSI job result and returns the preview data

func (*Job) GetPredictingResultPreview

func (job *Job) GetPredictingResultPreview() ([]string, [][]interface{}, int)

GetPredictingResultPreview returns a fraction of the predicting result data

func (*Job) GetTrainingResultSummary

func (job *Job) GetTrainingResultSummary() map[string]string

GetTrainingResultSummary returns the summary mapping of the training result. It is the evaluation info of the model

func (*Job) SubmitToFATE

func (job *Job) SubmitToFATE(finishCB func()) error

SubmitToFATE submits the job to FATE system and starts a monitoring routine

func (*Job) Update

func (job *Job) Update(newStatus *Job) error

Update updates the job info, including the fate job status. If the job starts running, a monitoring routine is started

func (*Job) UpdateResultInfo

func (job *Job) UpdateResultInfo(partyID uint, role JobParticipantRole) error

UpdateResultInfo gets the job result from FATE and updates it into the repo

func (*Job) UpdateStatus

func (job *Job) UpdateStatus(status JobStatus) error

UpdateStatus updates the job's status

func (*Job) UpdateStatusMessage

func (job *Job) UpdateStatusMessage(message string) error

UpdateStatusMessage updates the job's status message

func (*Job) Validate

func (job *Job) Validate() error

Validate checks the job configuration

type JobAlgorithmType

type JobAlgorithmType uint8

JobAlgorithmType is the enum of the job algorithm

const (
	JobAlgorithmTypeUnknown JobAlgorithmType = iota
	JobAlgorithmTypeHomoLR
	JobAlgorithmTypeHomoSBT
	JobAlgorithmTypeHeteroLR
	JobAlgorithmTypeHeteroSBT
)

type JobParticipant

type JobParticipant struct {
	gorm.Model
	UUID               string `gorm:"type:varchar(36)"`
	JobUUID            string `gorm:"type:varchar(36)"`
	SiteUUID           string `gorm:"type:varchar(36)"`
	SiteName           string `gorm:"type:varchar(255)"`
	SitePartyID        uint
	SiteRole           JobParticipantRole `gorm:"type:varchar(255)"`
	DataUUID           string             `gorm:"type:varchar(36)"`
	DataName           string             `gorm:"type:varchar(255)"`
	DataDescription    string             `gorm:"type:text"`
	DataTableName      string             `gorm:"type:varchar(255)"`
	DataTableNamespace string             `gorm:"type:varchar(255)"`
	DataLabelName      string             `gorm:"type:varchar(255)"`
	Status             JobParticipantStatus
	Repo               repo.JobParticipantRepository `gorm:"-"`
}

JobParticipant represents a site and its data for a job

func (*JobParticipant) Create

func (p *JobParticipant) Create() error

Create initialize the participant info and create it in the repo

func (*JobParticipant) UpdateStatus

func (p *JobParticipant) UpdateStatus(status JobParticipantStatus) error

UpdateStatus changes the participant's status

type JobParticipantRole

type JobParticipantRole string

JobParticipantRole is the enum of roles of a participant

const (
	JobParticipantRoleGuest JobParticipantRole = "guest"
	JobParticipantRoleHost  JobParticipantRole = "host"
)

type JobParticipantStatus

type JobParticipantStatus uint8

JobParticipantStatus is the status of this participant in the job

const (
	JobParticipantStatusUnknown JobParticipantStatus = iota
	JobParticipantStatusInitiator
	JobParticipantStatusPending
	JobParticipantStatusApproved
	JobParticipantStatusRejected
)

func (JobParticipantStatus) String

func (s JobParticipantStatus) String() string

type JobStatus

type JobStatus uint8

JobStatus is the enum of job status

const (
	JobStatusUnknown JobStatus = iota
	JobStatusPending
	JobStatusRejected
	JobStatusRunning
	JobStatusFailed
	JobStatusSucceeded
	JobStatusDeploying
	JobStatusDeleted
)

func (JobStatus) String

func (s JobStatus) String() string

type JobType

type JobType uint8

JobType is the enum of job type

const (
	JobTypeUnknown JobType = iota
	JobTypeTraining
	JobTypePredict
	JobTypePSI
)

func (JobType) String

func (t JobType) String() string

type LocalData

type LocalData struct {
	gorm.Model
	UUID string `gorm:"type:varchar(36);index;unique"`
	// Name is the name to reference the data
	Name string `gorm:"type:varchar(255);not null"`
	// Description contains more text about the data
	Description string `json:"description" gorm:"type:text"`
	// Column is a list of headers in this data
	Column Headers `json:"column" gorm:"type:text"`
	// TableName is the name of the data in FATE system
	TableName string `json:"table_name" gorm:"type:varchar(255)"`
	// TableNamespace is the namespace of the data in FATE system
	TableNamespace string `json:"table_namespace" gorm:"type:varchar(255)"`
	// Count is the number of the samples in the data
	Count uint64 `json:"count"`
	// Features is feature name list
	Features Headers `json:"feature_size" gorm:"type:text"`
	// Preview is the first 10 lines of data in this data
	Preview string `json:"preview" gorm:"type:text"`
	// IDMetaInfo is the meta data describing the ID column
	IDMetaInfo *valueobject.IDMetaInfo `json:"id_meta_info" gorm:"type:text;column:id_meta_info"`
	// JobID is the related FATE upload job id
	JobID string `json:"-" gorm:"type:varchar(255);column:job_id"`
	// JobConf is the related FATE upload job conf
	JobConf string `json:"-" gorm:"type:text;column:job_conf"`
	// JobStatus is the current status of the data in FATE system
	JobStatus UploadJobStatus `json:"status"`
	// JobErrorString records the error message if the job failed
	JobErrorMsg string `json:"job_error_msg" gorm:"type:text"`
	// LocalFilePath is the file path relative to the baseDir
	LocalFilePath string `json:"-" gorm:"type:varchar(255)"`
	// UploadContext contains info needed to finish the upload job
	UploadContext UploadContext `json:"-" gorm:"-"`
	// Repo is used to store the necessary data into the storage
	Repo repo.LocalDataRepository `json:"-" gorm:"-"`
}

LocalData represents an uploaded data file

func (*LocalData) ChangeJobStatus

func (d *LocalData) ChangeJobStatus(newStatus UploadJobStatus)

ChangeJobStatus upload the data's upload job status

func (*LocalData) CreateFromExistingTable

func (d *LocalData) CreateFromExistingTable() error

CreateFromExistingTable updates the local data info based on the existing table info in FATE and saves to the repo

func (*LocalData) Destroy

func (d *LocalData) Destroy() error

func (*LocalData) GetAbsFilePath

func (d *LocalData) GetAbsFilePath() (string, error)

GetAbsFilePath returns the absolute path the local date file

func (*LocalData) GetFlowDataDownloadRequest

func (d *LocalData) GetFlowDataDownloadRequest() (*http.Request, error)

GetFlowDataDownloadRequest returns the absolute path the local date file

func (*LocalData) UpdateIDMetaInfo

func (d *LocalData) UpdateIDMetaInfo(info *valueobject.IDMetaInfo) error

UpdateIDMetaInfo changes the meta info of the id column

func (*LocalData) Upload

func (d *LocalData) Upload(fileHeader *multipart.FileHeader) error

Upload save the file into local storage and uploaded it to FATE system

type Model

type Model struct {
	gorm.Model
	UUID                   string `gorm:"type:varchar(36)"`
	Name                   string `gorm:"type:varchar(255)"`
	FATEModelID            string `gorm:"type:varchar(255);column:fate_model_id"`
	FATEModelVersion       string `gorm:"type:varchar(255);column:fate_model_version"`
	ProjectUUID            string `gorm:"type:varchar(36)"`
	ProjectName            string `gorm:"type:varchar(255)"`
	JobUUID                string `gorm:"type:varchar(36)"`
	JobName                string `gorm:"type:varchar(255)"`
	ComponentName          string `gorm:"type:varchar(255)"`
	ComponentAlgorithmType ComponentAlgorithmType
	Role                   string `gorm:"type:varchar(255)"`
	PartyID                uint
	Evaluation             valueobject.ModelEvaluation `gorm:"type:text"`
	Repo                   repo.ModelRepository        `gorm:"-"`
}

Model is the domain entity of the model management context

func (*Model) Create

func (model *Model) Create() error

Create initializes the model and creates it in the repo

type ModelDeployment

type ModelDeployment struct {
	gorm.Model
	UUID                     string                         `json:"uuid" gorm:"type:varchar(36)"`
	ServiceName              string                         `json:"service_name" gorm:"type:varchar(255)"`
	ModelUUID                string                         `json:"model_uuid" gorm:"type:varchar(36)"`
	Type                     ModelDeploymentType            `json:"type"`
	Status                   ModelDeploymentStatus          `json:"status"`
	DeploymentParametersJson string                         `json:"parameters_json" gorm:"type:text"`
	RequestJson              string                         `json:"request_json" gorm:"type:text"`
	ResultJson               string                         `json:"result_json" gorm:"type:text"`
	Repo                     repo.ModelDeploymentRepository `json:"-" gorm:"-"`
}

ModelDeployment represents a deployment operation for a model

func (*ModelDeployment) Deploy

func (d *ModelDeployment) Deploy(context ModelDeploymentContext) error

Deploy deploys the model to FATE

type ModelDeploymentContext

type ModelDeploymentContext struct {
	Model              *Model
	FATEFlowContext    FATEFlowContext
	KubeflowConfig     valueobject.KubeflowConfig
	UserParametersJson string
}

ModelDeploymentContext contains the context needed to perform a deployment action

type ModelDeploymentStatus

type ModelDeploymentStatus uint8

ModelDeploymentStatus is a enum of the status of the deployment action

const (
	ModelDeploymentStatusUnknown ModelDeploymentStatus = iota
	ModelDeploymentStatusCreated
	ModelDeploymentStatusFailed
	ModelDeploymentStatusSucceeded
)

type ModelDeploymentType

type ModelDeploymentType uint

ModelDeploymentType is a enum of the types of the target deployment runtime We use uint instead of uint8 here because json marshalling will convert []uint8 slice to base64-encoded string

const (
	ModelDeploymentTypeUnknown ModelDeploymentType = iota
	ModelDeploymentTypeKFServing
)

func (ModelDeploymentType) String

func (t ModelDeploymentType) String() string

type Project

type Project struct {
	gorm.Model
	UUID string `gorm:"type:varchar(36);index;unique"`
	// Name is the name of the project
	Name string `gorm:"type:varchar(255);not null"`
	// Description contains more text about the project
	Description string `json:"description" gorm:"type:text"`
	// AutoApprovalEnabled is whether new jobs will be automatically approved
	AutoApprovalEnabled bool `json:"auto_approval_enabled"`
	// Type is the project type
	Type ProjectType
	// Status is the status of the project
	Status ProjectStatus
	// Creating/Managing site info
	valueobject.ProjectCreatorInfo
	// The repo for persistence
	Repo repo.ProjectRepository `json:"-" gorm:"-"`
}

Project is a container for federated machine learning jobs

func (*Project) Create

func (p *Project) Create() error

Create creates the project

type ProjectData

type ProjectData struct {
	gorm.Model
	Name           string                     `json:"name" gorm:"type:varchar(255)"`
	Description    string                     `json:"description" gorm:"type:text"`
	UUID           string                     `json:"uuid" gorm:"type:varchar(36)"`
	ProjectUUID    string                     `json:"project_uuid" gorm:"type:varchar(36)"`
	DataUUID       string                     `json:"data_uuid" gorm:"type:varchar(36)"`
	SiteUUID       string                     `json:"site_uuid" gorm:"type:varchar(36)"`
	SiteName       string                     `json:"site_name" gorm:"type:varchar(255)"`
	SitePartyID    uint                       `json:"site_party_id"`
	Type           ProjectDataType            `json:"type"`
	Status         ProjectDataStatus          `json:"status"`
	TableName      string                     `json:"table_name" gorm:"type:varchar(255)"`
	TableNamespace string                     `json:"table_namespace" gorm:"type:varchar(255)"`
	CreationTime   time.Time                  `json:"creation_time"`
	UpdateTime     time.Time                  `json:"update_time"`
	Repo           repo.ProjectDataRepository `json:"-" gorm:"-"`
}

ProjectData represents the data association in a project

type ProjectDataStatus

type ProjectDataStatus uint8

ProjectDataStatus is the status of this association

const (
	ProjectDataStatusUnknown ProjectDataStatus = iota
	ProjectDataStatusDismissed
	ProjectDataStatusAssociated
)

type ProjectDataType

type ProjectDataType uint8

ProjectDataType is the type of this association

const (
	ProjectDataTypeUnknown ProjectDataType = iota
	ProjectDataTypeLocal
	ProjectDataTypeRemote
)

type ProjectInvitation

type ProjectInvitation struct {
	gorm.Model
	UUID        string `gorm:"type:varchar(36);index;unique"`
	ProjectUUID string `gorm:"type:varchar(36)"`
	SiteUUID    string `gorm:"type:varchar(36)"`
	Status      ProjectInvitationStatus
}

ProjectInvitation is the invitation for a project

type ProjectInvitationStatus

type ProjectInvitationStatus uint8

ProjectInvitationStatus is the status of the invitation

const (
	// ProjectInvitationStatusCreated means the invitation is created but hasn't been sent yet
	ProjectInvitationStatusCreated ProjectInvitationStatus = iota
	ProjectInvitationStatusSent
	ProjectInvitationStatusRevoked
	ProjectInvitationStatusAccepted
	ProjectInvitationStatusRejected
)

type ProjectParticipant

type ProjectParticipant struct {
	gorm.Model
	UUID            string                   `json:"uuid" gorm:"type:varchar(36);index;unique"`
	ProjectUUID     string                   `json:"project_uuid" gorm:"type:varchar(36)"`
	SiteUUID        string                   `json:"site_uuid" gorm:"type:varchar(36)"`
	SiteName        string                   `json:"site_name" gorm:"type:varchar(255)"`
	SitePartyID     uint                     `json:"site_party_id"`
	SiteDescription string                   `json:"site_description"`
	Status          ProjectParticipantStatus `json:"status"`
}

ProjectParticipant is a site in a project

type ProjectParticipantStatus

type ProjectParticipantStatus uint8

ProjectParticipantStatus is the status of the site in a project

const (
	ProjectParticipantStatusUnknown ProjectParticipantStatus = iota
	ProjectParticipantStatusOwner
	ProjectParticipantStatusPending
	ProjectParticipantStatusJoined
	ProjectParticipantStatusRejected
	ProjectParticipantStatusLeft
	ProjectParticipantStatusDismissed
	ProjectParticipantStatusRevoked
)

type ProjectStatus

type ProjectStatus uint8

ProjectStatus is the status of a project

const (
	ProjectStatusManaged ProjectStatus = iota + 1
	ProjectStatusPending
	ProjectStatusJoined
	ProjectStatusRejected
	ProjectStatusLeft
	ProjectStatusClosed
	ProjectStatusDismissed
)

type ProjectType

type ProjectType uint8

ProjectType is the project type

const (
	ProjectTypeLocal ProjectType = iota + 1
	// ProjectTypeFederatedLocal means the project is locally created and is tracked in the FML manager
	ProjectTypeFederatedLocal
	ProjectTypeRemote
)

type Site

type Site struct {
	gorm.Model
	UUID string `json:"uuid" gorm:"type:varchar(36);index;unique"`
	// Name is the site's name
	Name string `json:"name" gorm:"type:varchar(255);unique;not null"`
	// Description contains more text about this site
	Description string `json:"description" gorm:"type:text"`
	// PartyID is the id of this party
	PartyID uint `json:"party_id" gorm:"column:party_id"`
	// ExternalHost is the IP or hostname this site portal service is exposed
	ExternalHost string `json:"external_host" gorm:"type:varchar(255);column:external_ip"`
	// ExternalPort the port number this site portal service is exposed
	ExternalPort uint `json:"external_port" gorm:"column:external_port"`
	// HTTPS choose if site portal has HTTPS enabled
	HTTPS bool `json:"https" gorm:"column:https"`
	// FMLManagerEndpoint is of format "<http or https>://<host>:<port>"
	FMLManagerEndpoint string `json:"fml_manager_endpoint" gorm:"type:varchar(255);column:fml_manager_endpoint"`
	// FMLManagerServerName is used to verify FML Manager's certificate
	FMLManagerServerName string `json:"fml_manager_server_name" gorm:"type:varchar(255);column:fml_manager_server_name"`
	// FMLManagerConnectedAt is the last time this portal has registered to a FML manager
	FMLManagerConnectedAt time.Time `json:"fml_manager_connected_at" gorm:"column:fml_manager_connected_at"`
	// FMLManagerConnected is whether the portal is connected to FML manager
	FMLManagerConnected bool `json:"fml_manager_connected" gorm:"column:fml_manager_connected"`
	// FATEFlowHost is the host address of the FATE-flow service
	FATEFlowHost string `json:"fate_flow_host" gorm:"type:varchar(255);column:fate_flow_host"`
	// FATEFlowHTTPPort is the http port number of the FATE-flow service
	FATEFlowHTTPPort uint `json:"fate_flow_http_port" gorm:"column:fate_flow_http_port"`
	// FATEFlowGRPCPort is the grpc port number of the FATE-flow service, currently not used
	FATEFlowGRPCPort uint `json:"fate_flow_grpc_port" gorm:"column:fate_flow_grpc_port"`
	// FATEFlowConnectedAt is the last time this portal connected to the FATE flow
	FATEFlowConnectedAt time.Time `json:"fate_flow_connected_at" gorm:"column:fate_flow_connected_at"`
	// FATEFlowConnected is whether the portal has connected to FATEFlow after the address is configured
	FATEFlowConnected bool `json:"fate_flow_connected" gorm:"column:fate_flow_connected"`
	// KubeflowConfig records the Kubeflow related information for deploying horizontal model to the KFServing system
	KubeflowConfig valueobject.KubeflowConfig `json:"kubeflow_config" gorm:"type:text;column:kubeflow_config"`
	// KubeflowConnectedAt is the last time this portal has successfully connected all related Kubeflow service
	KubeflowConnectedAt time.Time `json:"kubeflow_connected_at" gorm:"column:kubeflow_connected_at"`
	// KubeflowConnected is whether this site has connected to the Kubeflow since it is configured
	KubeflowConnected bool `json:"kubeflow_connected" gorm:"column:kubeflow_connected"`
	// Repo is the repository interface
	Repo repo.SiteRepository `json:"-" gorm:"-"`
}

Site contains all the info for the current site

func (*Site) ConnectFATEFlow

func (site *Site) ConnectFATEFlow(host string, port uint, https bool) error

ConnectFATEFlow try to issue a test request to the FATE-flow service

func (*Site) Load

func (site *Site) Load() error

Load site information from repository

func (*Site) RegisterToFMLManager

func (site *Site) RegisterToFMLManager(endpoint string, serverName string) error

RegisterToFMLManager registers this site to the FML manager service

func (*Site) UnregisterFromFMLManager

func (site *Site) UnregisterFromFMLManager() error

UnregisterFromFMLManager unregisters this site from the FML manager service

func (*Site) UpdateConfigurableInfo

func (site *Site) UpdateConfigurableInfo(updatedSite *Site) error

UpdateConfigurableInfo changes the site information

func (*Site) Validate

func (site *Site) Validate() error

Validate if the site has been properly configured

type UploadContext

type UploadContext struct {
	// FATEFlowHost is the host address of the service
	FATEFlowHost string
	// FATEFlowPort is the port of the service
	FATEFlowPort uint
	// FATEFlowIsHttps is whether the connection should be over TLS
	FATEFlowIsHttps bool
}

UploadContext currently contains FATE flow connection info

type UploadJobStatus

type UploadJobStatus uint8

UploadJobStatus is the status of the data in FATE system

const (
	UploadJobStatusToBeCreated UploadJobStatus = iota
	UploadJobStatusCreating
	UploadJobStatusRunning
	UploadJobStatusFailed
	UploadJobStatusSucceeded
)

func (*UploadJobStatus) MarshalJSON

func (s *UploadJobStatus) MarshalJSON() ([]byte, error)

MarshalJSON convert Cluster status to string

type User

type User struct {
	gorm.Model
	UUID string `gorm:"type:varchar(36);index;unique"`
	// Name is the user's name
	Name string `gorm:"type:varchar(255);unique;not null"`
	// Password is the user's hashed password
	Password string `gorm:"type:varchar(255)"`
	// PermissionInfo records the user's access to the system
	PermissionInfo valueobject.UserPermissionInfo `gorm:"embedded"`
	// Repo is the repository to persistent related data
	Repo repo.UserRepository `gorm:"-"`
}

User is a representation of the user available in the site

func (*User) CheckSitePortalAccess

func (u *User) CheckSitePortalAccess() error

CheckSitePortalAccess returns error if the user doesn't have the portal access

func (*User) LoadById

func (u *User) LoadById() error

LoadById reads the info from the repo

func (*User) UpdatePermissionInfo

func (u *User) UpdatePermissionInfo(info valueobject.UserPermissionInfo) error

UpdatePermissionInfo changes the user's permission

func (*User) UpdatePwdInfo

func (u *User) UpdatePwdInfo(curPassword, newPassword string) error

UpdatePwdInfo updates a user's password

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL