distcc

Documentation

Index

• Constants
• func GetK8sInstanceKey(queueName string) *config.InstanceType
• func NewDistccEngine(conf EngineConfig, mgr, k8sCrmMgr crm.HandlerWithUser, ...) (engine.Engine, error)
• type CCacheStats
  • func (cs CCacheStats) Dump() []byte
• type CombinedProject
• type CommandSetting
  • func (cs *CommandSetting) GetCommand() string
  • func (cs *CommandSetting) GetCompiler() (string, string)
• type CommandType
• type CustomData
• type DeltaInfoStats
• type EngineConfig
• type ExtraData
• type ExtraVars
• type K8sClusterInfo
• type Message
• type MessageGetCMakeArgs
• type MessageRecordStats
• type MessageResponse
  • func (mr *MessageResponse) Dump() []byte
• type MessageType
• type MySQL
  • func NewMySQL(conf engine.MySQLConf) (MySQL, error)
• type SummaryResult
• type TableGcc
  • func (tg *TableGcc) CheckData() error
  • func (tg TableGcc) TableName() string
• type TableProjectInfo
  • func (tpi TableProjectInfo) TableName() string
• type TableProjectSetting
  • func (tps TableProjectSetting) TableName() string
• type TableTask
  • func (tt TableTask) TableName() string
• type TableWhitelist
  • func (twl TableWhitelist) TableName() string

Constants

const (

	// EngineName define the engine name
	EngineName = "distcc"
)

Functions

func GetK8sInstanceKey

func GetK8sInstanceKey(queueName string) *config.InstanceType

GetK8sInstanceKey get instance type from queueName

func NewDistccEngine

func NewDistccEngine(conf EngineConfig, mgr, k8sCrmMgr crm.HandlerWithUser,
	k8sListCrmMgr map[string]crm.HandlerWithUser) (engine.Engine, error)

NewDistccEngine get a new distcc engine EngineConfig: describe the basic config of engine including mysql config HandleWithUser: registered from a container resource manager, used to handle the resources and launch tasks.

Types

type CCacheStats

type CCacheStats struct {
	CacheDir                  string `json:"cache_dir"`
	PrimaryConfig             string `json:"primary_config"`
	SecondaryConfig           string `json:"secondary_config"`
	DirectHit                 int    `json:"cache_direct_hit"`
	PreprocessedHit           int    `json:"cache_preprocessed_hit"`
	CacheMiss                 int    `json:"cache_miss"`
	CalledForLink             int    `json:"called_for_link"`
	CalledForPreProcessing    int    `json:"called_for_processing"`
	UnsupportedSourceLanguage int    `json:"unsupported_source_language"`
	NoInputFile               int    `json:"no_input_file"`
	FilesInCache              int    `json:"files_in_cache"`
	CacheSize                 string `json:"cache_size"`
	MaxCacheSize              string `json:"max_cache_size"`
}

CCacheStats describe the ccache stats data from 'ccache -s'.

func (CCacheStats) Dump

func (cs CCacheStats) Dump() []byte

dump the struct data into byte

type CombinedProject

type CombinedProject struct {
	*TableProjectSetting
	*TableProjectInfo
}

CombinedProject generate project_settings and project_records

type CommandSetting

type CommandSetting struct {
	CommandType CommandType
	Command     string

	CompilerVersion string
	CPUNum          int
	CustomParam     string
	LeastJobServer  int

	CCacheEnable  bool
	BanDistCC     bool
	BanAllBooster bool

	Extra taskClientExtra
}

CommandSetting contains command settings

func (*CommandSetting) GetCommand

func (cs *CommandSetting) GetCommand() string

GetCommand get command or args by settings, this command will be executed directly in client side.

func (*CommandSetting) GetCompiler

func (cs *CommandSetting) GetCompiler() (string, string)

GetCompiler get the compiler param.

type CommandType

type CommandType string

const (
	CommandMake  CommandType = "make"
	CommandCmake CommandType = "cmake"
	CommandBazel CommandType = "bazel"
	CommandBlade CommandType = "blade"
	CommandNinja CommandType = "ninja"
)

type CustomData

type CustomData struct {
	GccVersion        string            `json:"gcc_version"`
	Commands          string            `json:"commands"`
	Environments      map[string]string `json:"environments"`
	UnsetEnvironments []string          `json:"unset_environments"`
	CCacheEnable      bool              `json:"ccache_enable"`
	CCCompiler        string            `json:"cc_compiler"`
	CXXCompiler       string            `json:"cxx_compiler"`
	JobServer         int               `json:"job_server"`
	DistCCHosts       string            `json:"distcc_hosts"`
}

CustomData describe the detail data of distcc task.

type DeltaInfoStats

type DeltaInfoStats struct {
	CompileFilesOK      int64
	CompileFilesErr     int64
	CompileFilesTimeout int64
	CompileUnits        float64
}

DeltaInfoStats describe the project info delta data.

type EngineConfig

type EngineConfig struct {
	engine.MySQLConf
	Rd rd.RegisterDiscover

	// k8s cluster info
	K8SCRMClusterID      string
	K8SCRMCPUPerInstance float64
	K8SCRMMemPerInstance float64

	// k8s cluster list info
	K8SClusterList map[string]K8sClusterInfo

	ClusterID      string
	CPUPerInstance float64
	MemPerInstance float64

	LeastJobServer         int
	JobServerTimesToCPU    float64
	QueueShareType         map[string]engine.QueueShareType
	Brokers                []config.EngineDistCCBrokerConfig
	QueueResourceAllocater map[string]config.ResourceAllocater
}

EngineConfig define distcc engine

type ExtraData

type ExtraData struct {
	User          string `json:"user"`
	GccVersion    string `json:"gcc_version"`
	RunDir        string `json:"run_dir"`
	Params        string `json:"params"` //自定义参数
	CCacheEnabled *bool  `json:"ccache_enabled"`

	// command define the target to be called, such as make, bazel, /data/custom/make etc.
	Command     string      `json:"command,omitempty"`
	CommandType CommandType `json:"command_type,omitempty"`

	// extra_vars includes the extra params need by client
	ExtraVars ExtraVars `json:"extra_vars,omitempty"`
}

ExtraData describe the data in task creation from client.

type ExtraVars

type ExtraVars struct {
	// bazelrc define the bazelrc file path
	BazelRC string `json:"bazelrc"`
	MaxJobs int    `json:"max_jobs,omitempty"`
}

ExtraVars describe the extra params in ExtraData

type K8sClusterInfo

type K8sClusterInfo struct {
	K8SCRMClusterID      string
	K8SCRMCPUPerInstance float64
	K8SCRMMemPerInstance float64
}

K8sClusterInfo define

type Message

type Message struct {
	Type MessageType `json:"type"`

	MessageRecordStats  MessageRecordStats  `json:"record_stats"`
	MessageGetCMakeArgs MessageGetCMakeArgs `json:"get_cmake_args"`
}

Message describe the data format from SendMessage caller.

type MessageGetCMakeArgs

type MessageGetCMakeArgs struct {
	CCacheEnabled *bool `json:"ccache_enabled"`
}

MessageGetCMakeArgs describe the message data of type cmake args.

type MessageRecordStats

type MessageRecordStats struct {
	Message     string      `json:"message"`
	CCacheStats CCacheStats `json:"ccache_stats"`
}

MessageRecordStats describe the message data of type record stats.

type MessageResponse

type MessageResponse struct {
	CMakeArgs string `json:"cmake_args"`
}

MessageResponse describe the return data of SendMessage

func (*MessageResponse) Dump

func (mr *MessageResponse) Dump() []byte

Dump dump the struct dat into byte

type MessageType

type MessageType int

MessageType define

const (
	// MessageTypeRecordStats means this message is about record stats from client.
	MessageTypeRecordStats MessageType = iota

	// MessageTypeGetCMakeArgs means this message is about to get cmake args.
	MessageTypeGetCMakeArgs
)

type MySQL

type MySQL interface {
	// get db operator
	GetDB() *gorm.DB

	ListTask(opts commonMySQL.ListOptions) ([]*TableTask, int64, error)
	GetTask(taskID string) (*TableTask, error)
	PutTask(task *TableTask) error
	UpdateTask(taskID string, task map[string]interface{}) error
	DeleteTask(taskID string) error

	ListProject(opts commonMySQL.ListOptions) ([]*CombinedProject, int64, error)

	ListProjectInfo(opts commonMySQL.ListOptions) ([]*TableProjectInfo, int64, error)
	GetProjectInfo(projectID string) (*TableProjectInfo, error)
	PutProjectInfo(projectInfo *TableProjectInfo) error
	UpdateProjectInfo(projectID string, projectInfo map[string]interface{}) error
	DeleteProjectInfo(projectID string) error
	AddProjectInfoStats(projectID string, delta DeltaInfoStats) error

	ListProjectSetting(opts commonMySQL.ListOptions) ([]*TableProjectSetting, int64, error)
	GetProjectSetting(projectID string) (*TableProjectSetting, error)
	PutProjectSetting(projectSetting *TableProjectSetting) error
	UpdateProjectSetting(projectID string, projectSetting map[string]interface{}) error
	DeleteProjectSetting(projectID string) error
	CreateOrUpdateProjectSetting(projectSetting *TableProjectSetting, projectSettingRaw map[string]interface{}) error

	ListWhitelist(opts commonMySQL.ListOptions) ([]*TableWhitelist, int64, error)
	GetWhitelist(key engine.WhiteListKey) (*TableWhitelist, error)
	PutWhitelist(wll []*TableWhitelist) error
	UpdateWhitelist(key engine.WhiteListKey, wll []map[string]interface{}) error
	DeleteWhitelist(keys []*engine.WhiteListKey) error

	ListGcc(opts commonMySQL.ListOptions) ([]*TableGcc, int64, error)
	GetGcc(gccVersion string) (*TableGcc, error)
	PutGcc(gcc *TableGcc) error
	UpdateGcc(gccVersion string, gcc map[string]interface{}) error
	DeleteGcc(gccVersion string) error

	SummaryTaskRecords(opts commonMySQL.ListOptions) ([]*SummaryResult, int64, error)
}

MySQL describe the full operations to mysql databases need by engine.

func NewMySQL

func NewMySQL(conf engine.MySQLConf) (MySQL, error)

NewMySQL get new mysql instance with connected orm operator.

type SummaryResult

type SummaryResult struct {
	Day               string  `json:"day"`
	ProjectID         string  `json:"project_id"`
	TotalTime         float64 `json:"total_time"`
	TotalTimeWithCPU  float64 `json:"total_time_with_cpu"`
	TotalRecordNumber int     `json:"total_record_number"`
}

SummaryResult generate summary data

type TableGcc

type TableGcc struct {
	engine.TableBasic

	GccVersion string `gorm:"column:gcc_version;primary_key" json:"gcc_version"`
	Image      string `gorm:"column:image" sql:"type:text" json:"image"`
}

TableGcc describe the db columns of gcc. It inherit directly from Basic.

func (*TableGcc) CheckData

func (tg *TableGcc) CheckData() error

CheckData check if gcc data valid.

func (TableGcc) TableName

func (tg TableGcc) TableName() string

TableName specific table name.

type TableProjectInfo

type TableProjectInfo struct {
	engine.TableProjectInfoBasic

	CompileFilesOK      int64   `gorm:"column:compile_files_ok" json:"compile_files_ok"`
	CompileFilesErr     int64   `gorm:"column:compile_files_err" json:"compile_files_err"`
	CompileFilesTimeout int64   `gorm:"column:compile_files_timeout" json:"compile_files_timeout"`
	CompileUnits        float64 `gorm:"column:compile_units" json:"compile_units"`
}

TableProjectInfo describe the db columns of project info. It will inherit the ProjectInfoBasic.

func (TableProjectInfo) TableName

func (tpi TableProjectInfo) TableName() string

TableName specific table name.

type TableProjectSetting

type TableProjectSetting struct {
	engine.TableProjectBasic

	RequestCPU    float64 `gorm:"column:request_cpu" json:"request_cpu"`
	LeastCPU      float64 `gorm:"column:least_cpu" json:"least_cpu"`
	SuggestCPU    float64 `gorm:"column:suggest_cpu;default:0" json:"suggest_cpu"`
	AcceptedTime  int64   `gorm:"column:accepted_time;default:0" json:"accepted_time"`
	City          string  `gorm:"column:city" json:"city"`
	GccVersion    string  `gorm:"column:gcc_version" json:"gcc_version"`
	CCacheEnabled bool    `gorm:"column:ccache_enabled" json:"ccache_enabled"`
	BanDistCC     bool    `gorm:"column:ban_distcc;default:false" json:"ban_distcc"`
	BanAllBooster bool    `gorm:"column:ban_all_booster;default:false" json:"ban_all_booster"`

	// extra field, no includes under the distCC-server management, but for distCC-controller
	BanController bool `gorm:"column:ban_controller;default:false" json:"ban_controller"`
}

TableProjectSetting describe the db columns of project setting. It will inherit the ProjectBasic.

func (TableProjectSetting) TableName

func (tps TableProjectSetting) TableName() string

TableName specific table name.

type TableTask

type TableTask struct {
	engine.TableTaskBasic

	// client
	City          string  `gorm:"column:city" json:"city"`
	SourceIP      string  `gorm:"column:source_ip" json:"source_ip"`
	SourceCPU     int     `gorm:"column:source_cpu" json:"source_cpu"`
	GccVersion    string  `gorm:"column:gcc_version" json:"gcc_version"`
	User          string  `gorm:"column:user" json:"user"`
	Params        string  `gorm:"column:params" sql:"type:text" json:"params"`
	Cmd           string  `gorm:"column:cmd" sql:"type:text" json:"cmd"`
	Env           string  `gorm:"column:env" sql:"type:text" json:"env"`
	RequestCPU    float64 `gorm:"column:request_cpu" json:"request_cpu"`
	LeastCPU      float64 `gorm:"column:least_cpu" json:"least_cpu"`
	CCacheEnabled bool    `gorm:"column:ccache_enabled" json:"ccache_enabled"`
	BanDistCC     bool    `gorm:"column:ban_distcc;default:false" json:"ban_distcc"`
	BanAllBooster bool    `gorm:"column:ban_all_booster;default:false" json:"ban_all_booster"`
	RunDir        string  `gorm:"column:run_dir" sql:"type:text" json:"run_dir"`
	CommandType   string  `gorm:"column:command_type" json:"command_type"`
	Command       string  `gorm:"column:command" sql:"type:text" json:"command"`
	Extra         string  `gorm:"column:extra" sql:"type:text" json:"extra"`

	// compilers
	Compilers string `gorm:"column:compilers" sql:"type:text" json:"compilers"`

	// status
	CompileFilesOK       int64   `gorm:"column:compile_files_ok" json:"compile_files_ok"`
	CompileFilesErr      int64   `gorm:"column:compile_files_err" json:"compile_files_err"`
	CompileFilesTimeout  int64   `gorm:"column:compile_files_timeout" json:"compile_files_timeout"`
	CompilerCount        int     `gorm:"column:compiler_count" json:"compiler_count"`
	CPUTotal             float64 `gorm:"column:cpu_total" json:"cpu_total"`
	MemTotal             float64 `gorm:"column:mem_total" json:"mem_total"`
	CCacheInfo           string  `gorm:"column:ccache_info" sql:"type:text" json:"ccache_info"`
	CacheDirectHit       int64   `gorm:"column:cache_direct_hit" json:"cache_direct_hit"`
	CachePreprocessedHit int64   `gorm:"column:cache_preprocessed_hit" json:"cache_preprocessed_hit"`
	CacheMiss            int64   `gorm:"column:cache_miss" json:"cache_miss"`
	FilesInCache         int64   `gorm:"column:files_in_cache" json:"files_in_cache"`
	CacheSize            string  `gorm:"column:cache_size" json:"cache_size"`
	MaxCacheSize         string  `gorm:"column:max_cache_size" json:"max_cache_size"`

	// operator
	ClusterID         string  `gorm:"column:cluster_id" json:"cluster_id"`
	AppName           string  `gorm:"column:app_name" json:"app_name"`
	Namespace         string  `gorm:"column:namespace" json:"namespace"`
	Image             string  `gorm:"column:image" sql:"type:text" json:"image"`
	Instance          int     `gorm:"column:instance" json:"instance"`
	RequestInstance   int     `gorm:"column:request_instance" json:"request_instance"`
	LeastInstance     int     `gorm:"column:least_instance" json:"least_instance"`
	RequestCPUPerUnit float64 `gorm:"column:request_cpu_per_unit" json:"request_cpu_per_unit"`
	RequestMemPerUnit float64 `gorm:"column:request_mem_per_unit" json:"request_mem_per_unit"`

	// extra field, no includes under the distcc-server management, but for distcc-controller
	MaxJobs  int64 `gorm:"column:max_jobs;default:0" json:"max_jobs"`
	Observed bool  `gorm:"column:observed;default:false;index" json:"observed"`
}

TableTask describe the db columns of Task. It will inherit the TaskBasic.

func (TableTask) TableName

func (tt TableTask) TableName() string

TableName specific table name.

type TableWhitelist

type TableWhitelist struct {
	engine.TableWhitelistBasic
}

TableWhitelist describe the db columns of whitelist. It will inherit the WhitelistBasic.

func (TableWhitelist) TableName

func (twl TableWhitelist) TableName() string

TableName specific table name.