apisjob

package
v0.0.0-...-3f967bb Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 15, 2025 License: Apache-2.0, BSD-3-Clause, MIT Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// define the engine name
	EngineName = "apisjob"
)

Variables

This section is empty.

Functions

func NewApisEngine 

func NewApisEngine(
	conf EngineConfig,
	k8sCrmMgr crm.HandlerWithUser,
	directMgr direct.HandleWithUser) (engine.Engine, error)

NewApisEngine get a new apis engine EngineConfig: describe the basic config of engine including mysql config HandleWithUser: registered from a direct resource manager, used to handle the resources and launch tasks.

Types

type CombinedProject 

type CombinedProject struct {
	*TableProjectSetting
	*TableProjectInfo
}

CombinedProject generate project_settings and project_records

type DeltaInfoStats 

type DeltaInfoStats struct {
	ServiceUnits float64
}

DeltaInfoStats describe the project info delta data.

type EngineConfig 

type EngineConfig struct {
	engine.MySQLConf
	QueueResourceAllocater map[string]config.ResourceAllocater

	// k8s cluster info
	K8SCRMClusterID      string
	K8SCRMCPUPerInstance float64
	K8SCRMMemPerInstance float64
}

EngineConfig

type ExtraData 

type ExtraData struct {
	ProjectID string `json:"project_id"`
}

extra data protocol, should be decode when creating task

type ExtraInfo 

type ExtraInfo struct {
	MaxCores int `json:"max_cores"`
}

ExtraInfo describe the data dump from task.

type Message 

type Message struct {
	Type        MessageType        `json:"type"`
	RecordStats MessageRecordStats `json:"stats"`
}

Message describe the data format from SendTaskMessage caller.

type MessageRecordStats 

type MessageRecordStats struct {
	CompleteTasks int           `json:"complete_tasks"`
	FailedTasks   int           `json:"failed_tasks"`
	Agents        []interface{} `json:"agents"`
}

MessageRecordStats describe the stats data from client's report

type MessageType 

type MessageType string
const (
	// MessageTypeRecordStats means this message is about stats info from client.
	MessageTypeRecordStats MessageType = "stats"
)

type MySQL 

type MySQL interface {
	// get db operator
	GetDB() *gorm.DB

	ListTask(opts commonMySQL.ListOptions) ([]*TableTask, int64, error)
	GetTask(taskID string) (*TableTask, error)
	PutTask(task *TableTask) error
	UpdateTask(taskID string, task map[string]interface{}) error
	DeleteTask(taskID string) error

	ListProject(opts commonMySQL.ListOptions) ([]*CombinedProject, int64, error)

	ListProjectInfo(opts commonMySQL.ListOptions) ([]*TableProjectInfo, int64, error)
	GetProjectInfo(projectID string) (*TableProjectInfo, error)
	PutProjectInfo(projectInfo *TableProjectInfo) error
	UpdateProjectInfo(projectID string, projectInfo map[string]interface{}) error
	DeleteProjectInfo(projectID string) error
	AddProjectInfoStats(projectID string, delta DeltaInfoStats) error

	ListProjectSetting(opts commonMySQL.ListOptions) ([]*TableProjectSetting, int64, error)
	GetProjectSetting(projectID string) (*TableProjectSetting, error)
	PutProjectSetting(projectSetting *TableProjectSetting) error
	UpdateProjectSetting(projectID string, projectSetting map[string]interface{}) error
	DeleteProjectSetting(projectID string) error
	CreateOrUpdateProjectSetting(projectSetting *TableProjectSetting, projectSettingRaw map[string]interface{}) error

	ListWhitelist(opts commonMySQL.ListOptions) ([]*TableWhitelist, int64, error)
	GetWhitelist(key engine.WhiteListKey) (*TableWhitelist, error)
	PutWhitelist(wll []*TableWhitelist) error
	UpdateWhitelist(key engine.WhiteListKey, wll []map[string]interface{}) error
	DeleteWhitelist(keys []*engine.WhiteListKey) error
}

MySQL describe the full operations to mysql databases need by engine.

func NewMySQL 

func NewMySQL(conf engine.MySQLConf) (MySQL, error)

NewMySQL get new mysql instance with connected orm operator.

type TableProjectInfo 

type TableProjectInfo struct {
	engine.TableProjectInfoBasic

	ServiceUnits float64 `gorm:"column:service_units" json:"service_units"`
}

TableProjectInfo describe the db columns of project info. It will inherit the ProjectInfoBasic.

func (TableProjectInfo) TableName 

func (tpi TableProjectInfo) TableName() string

TableName specific table name.

type TableProjectSetting 

type TableProjectSetting struct {
	engine.TableProjectBasic

	RequestCPU float64 `gorm:"column:request_cpu" json:"request_cpu"`
	LeastCPU   float64 `gorm:"column:least_cpu" json:"least_cpu"`

	// agent worker parameters
	CoordinatorIP            string `gorm:"column:coordinator_ip" json:"coordinator_ip"`
	CoordinatorPort          int    `gorm:"column:coordinator_port" json:"coordinator_port"`
	AgentCommandName         string `gorm:"column:agent_command_name" json:"agent_command_name"`
	AgentCommandPath         string `gorm:"column:agent_command_path" sql:"type:text" json:"agent_command_path"`
	AgentCommandDir          string `gorm:"column:agent_command_dir" sql:"type:text" json:"agent_command_dir"`
	AgentPort                int    `gorm:"column:agent_port" json:"agent_port"`
	AgentFileServerPort      int    `gorm:"column:agent_file_server_port" json:"agent_file_server_port"`
	AgentNTriesConnectingRPC int    `gorm:"column:agent_n_tries_connecting_rpc" json:"agent_n_tries_connecting_rpc"`
	AgentCachePath           string `gorm:"column:agent_cache_path" sql:"type:text" json:"agent_cache_path"`
	Image                    string `gorm:"column:image" sql:"type:text" json:"image"`
	CacheIP                  string `gorm:"column:cache_ip" json:"cache_ip"`
	CachePort                int    `gorm:"column:cache_port" json:"cache_port"`
	UseGdt                   bool   `gorm:"column:use_gdt" json:"use_gdt"`
	VolumeMounts             string `gorm:"column:volume_mounts" sql:"type:text" json:"volume_mounts"`

	// following fields all deprecated, but will be kept in database table as columns.
	AgentMaxTasksPerJob           int   `gorm:"column:agent_max_tasks_per_job" json:"agent_max_tasks_per_job"`
	AgentMaxTasks                 int   `gorm:"column:agent_max_tasks" json:"agent_max_tasks"`
	AgentKeepAliveInterval        int   `gorm:"column:agent_keep_alive_interval" json:"agent_keep_alive_interval"`
	AgentDependPreparationTimeout int   `gorm:"column:agent_depend_preparation_timeout" json:"agent_depend_preparation_timeout"`
	AgentClientKeepAliveTimeout   int   `gorm:"column:agent_client_keep_alive_timeout" json:"agent_client_keep_alive_timeout"`
	AgentFileRetrievingRetryTimes int   `gorm:"column:agent_file_retrieving_retry_times" json:"agent_file_retrieving_retry_times"`
	AgentFileRequestTimeout       int   `gorm:"column:agent_file_request_timeout" json:"agent_file_request_timeout"`
	AgentMaxWorkingProcesses      int   `gorm:"column:agent_max_working_processes" json:"agent_max_working_processes"`
	AgentMaxWorkingCapability     int   `gorm:"column:agent_max_working_capability" json:"agent_max_working_capability"`
	AgentMaxCores                 int   `gorm:"column:agent_max_cores" json:"agent_max_cores"`
	AgentMaxFileRetrievingCount   int   `gorm:"column:agent_max_file_retrieving_count" json:"agent_max_file_retrieving_count"`
	AgentNoCoordinator            bool  `gorm:"column:agent_no_coordinator;default:false" json:"agent_no_coordinator"`
	AgentFilePackageSize          int64 `gorm:"column:agent_file_package_size" json:"agent_file_package_size"`
	AgentChunkRetryInterval       int   `gorm:"column:agent_chunk_retry_interval" json:"agent_chunk_retry_interval"`
	AgentMaxTransmissionPeers     int   `gorm:"column:agent_max_transmission_peers" json:"agent_max_transmission_peers"`
	AgentMaxActiveTransmission    int   `gorm:"column:agent_max_active_transmission" json:"agent_max_active_transmission"`
	AgentEnableFileCompress       bool  `gorm:"column:agent_enable_file_compress" json:"agent_enable_file_compress"`
	AgentPrepareTimeout           int   `gorm:"column:agent_prepare_timeout" json:"agent_prepare_timeout"`
	AgentEnableFileLog            bool  `gorm:"column:agent_enable_file_log" json:"agent_enable_file_log"`
}

TableProjectSetting describe the db columns of project setting. It will inherit the ProjectBasic.

func (TableProjectSetting) TableName 

func (tps TableProjectSetting) TableName() string

TableName specific table name.

type TableTask 

type TableTask struct {
	engine.TableTaskBasic

	ResourceID string  `gorm:"column:resource_id" json:"resource_id"`
	RequestCPU float64 `gorm:"column:request_cpu" json:"request_cpu"`
	LeastCPU   float64 `gorm:"column:least_cpu" json:"least_cpu"`
	CPUTotal   float64 `gorm:"column:cpu_total" json:"cpu_total"`
	MemTotal   float64 `gorm:"column:mem_total" json:"mem_total"`

	// agent worker project_id, specified by client
	AgentProjectID string `gorm:"column:agent_project_id" json:"agent_project_id"`
	WorkerIPList   string `gorm:"column:worker_ip_list" sql:"type:text" json:"worker_ip_list"`

	// stats from client
	CompleteTasks int    `gorm:"column:complete_tasks" json:"complete_tasks"`
	FailedTasks   int    `gorm:"column:failed_tasks" json:"failed_tasks"`
	AgentsInfo    string `gorm:"column:agents_info" sql:"type:text" json:"agents_info"`

	// container settings
	Image           string  `gorm:"column:image" sql:"type:text" json:"image"`
	CPUPerInstance  float64 `gorm:"cpu_per_instance" json:"cpu_per_instance"`
	MemPerInstance  float64 `gorm:"mem_per_instance" json:"mem_per_instance"`
	Instance        int     `gorm:"instance" json:"instance"`
	LeastInstance   int     `gorm:"least_instance" json:"least_instance"`
	RequestInstance int     `gorm:"request_instance" json:"request_instance"`

	// set from project settings
	CoordinatorIP            string `gorm:"column:coordinator_ip" json:"coordinator_ip"`
	CoordinatorPort          int    `gorm:"column:coordinator_port" json:"coordinator_port"`
	AgentCommandName         string `gorm:"column:agent_command_name" json:"agent_command_name"`
	AgentCommandPath         string `gorm:"column:agent_command_path" sql:"type:text" json:"agent_command_path"`
	AgentCommandDir          string `gorm:"column:agent_command_dir" sql:"type:text" json:"agent_command_dir"`
	AgentPort                int    `gorm:"column:agent_port" json:"agent_port"`
	AgentFileServerPort      int    `gorm:"column:agent_file_server_port" json:"agent_file_server_port"`
	AgentNTriesConnectingRPC int    `gorm:"column:agent_n_tries_connecting_rpc" json:"agent_n_tries_connecting_rpc"`
	AgentCachePath           string `gorm:"column:agent_cache_path" sql:"type:text" json:"agent_cache_path"`
	CacheIP                  string `gorm:"column:cache_ip" json:"cache_ip"`
	CachePort                int    `gorm:"column:cache_port" json:"cache_port"`
	UseGdt                   bool   `gorm:"column:use_gdt" json:"use_gdt"`
	VolumeMounts             string `gorm:"column:volume_mounts" sql:"type:text" json:"volume_mounts"`

	// following fields all deprecated, but will be kept in database table as columns.
	AgentMaxTasksPerJob           int   `gorm:"column:agent_max_tasks_per_job" json:"agent_max_tasks_per_job"`
	AgentMaxTasks                 int   `gorm:"column:agent_max_tasks" json:"agent_max_tasks"`
	AgentKeepAliveInterval        int   `gorm:"column:agent_keep_alive_interval" json:"agent_keep_alive_interval"`
	AgentDependPreparationTimeout int   `gorm:"column:agent_depend_preparation_timeout" json:"agent_depend_preparation_timeout"`
	AgentClientKeepAliveTimeout   int   `gorm:"column:agent_client_keep_alive_timeout" json:"agent_client_keep_alive_timeout"`
	AgentFileRetrievingRetryTimes int   `gorm:"column:agent_file_retrieving_retry_times" json:"agent_file_retrieving_retry_times"`
	AgentFileRequestTimeout       int   `gorm:"column:agent_file_request_timeout" json:"agent_file_request_timeout"`
	AgentMaxWorkingProcesses      int   `gorm:"column:agent_max_working_processes" json:"agent_max_working_processes"`
	AgentMaxWorkingCapability     int   `gorm:"column:agent_max_working_capability" json:"agent_max_working_capability"`
	AgentMaxCores                 int   `gorm:"column:agent_max_cores" json:"agent_max_cores"`
	AgentMaxFileRetrievingCount   int   `gorm:"column:agent_max_file_retrieving_count" json:"agent_max_file_retrieving_count"`
	AgentNoCoordinator            bool  `gorm:"column:agent_no_coordinator;default:true" json:"agent_no_coordinator"`
	AgentFilePackageSize          int64 `gorm:"column:agent_file_package_size" json:"agent_file_package_size"`
	AgentChunkRetryInterval       int   `gorm:"column:agent_chunk_retry_interval" json:"agent_chunk_retry_interval"`
	AgentMaxTransmissionPeers     int   `gorm:"column:agent_max_transmission_peers" json:"agent_max_transmission_peers"`
	AgentMaxActiveTransmission    int   `gorm:"column:agent_max_active_transmission" json:"agent_max_active_transmission"`
	AgentEnableFileCompress       bool  `gorm:"column:agent_enable_file_compress" json:"agent_enable_file_compress"`
	AgentPrepareTimeout           int   `gorm:"column:agent_prepare_timeout" json:"agent_prepare_timeout"`
	AgentEnableFileLog            bool  `gorm:"column:agent_enable_file_log" json:"agent_enable_file_log"`
}

TableTask describe the db columns of Task. It will inherit the TaskBasic.

func (TableTask) TableName 

func (tt TableTask) TableName() string

specific table name.

type TableWhitelist 

type TableWhitelist struct {
	engine.TableWhitelistBasic
}

TableWhitelist describe the db columns of whitelist. It will inherit the WhitelistBasic.

func (TableWhitelist) TableName 

func (twl TableWhitelist) TableName() string

TableName specific table name.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.