hatchery

Documentation

Index

• Constants
• Variables
• func CheckWorkerModelRegister(ctx context.Context, h Interface, modelPath string) error
• func Create(ctx context.Context, h Interface) error
• func InitMetrics(ctx context.Context) error
• func ModelInterpolateSecrets(hWithModels InterfaceWithModels, model *sdk.Model) error
• func NewWorkerToken(hatcheryName string, privateKey *rsa.PrivateKey, expiration time.Time, ...) (string, error)
• func ParseArgs(str string) []string
• func ParseRequirementModel(rm string) (string, map[string]string)
• func SendSpawnInfo(ctx context.Context, h Interface, jobID int64, spawnMsg sdk.SpawnMsg)
• func WorkerPool(ctx context.Context, h Interface, statusFilter ...string) ([]sdk.Worker, error)
• type CacheNbAttemptsJobIDs
  • func (c *CacheNbAttemptsJobIDs) Key(id int64) string
  • func (c *CacheNbAttemptsJobIDs) NewAttempt(id int64) int
• type Interface
• type InterfaceWithModels
• type JobIdentifiers
• type Metrics
  • func GetMetrics() *Metrics
• type SpawnArguments
  • func (s *SpawnArguments) ModelName() string
• type WorkerJWTClaims

Constants

const (
	LabelServiceJobID        = "CDS_JOB_ID"
	LabelServiceProjectKey   = "CDS_PROJECT_KEY"
	LabelServiceWorkflowName = "CDS_WORKFLOW_NAME"
	LabelServiceWorkflowID   = "CDS_WORKFLOW_ID"
	LabelServiceRunID        = "CDS_WORKFLOW_RUN_ID"
	LabelServiceNodeRunName  = "CDS_NODE_RUN_NAME"
	LabelServiceNodeRunID    = "CDS_NODE_RUN_ID"
	LabelServiceJobName      = "CDS_JOB_NAME"
	LabelServiceID           = "CDS_SERVICE_ID"
	LabelServiceReqName      = "CDS_SERVICE_NAME"
)

const MemoryRegisterContainer int64 = 128

MemoryRegisterContainer is the RAM used for spawning a docker container for register a worker model. 128 Mo

Variables

var (
	// Client is a CDS Client
	Client cdsclient.HTTPClient

	CacheSpawnIDsTTL      = 10 * time.Second
	CacheNbAttemptsIDsTTL = 1 * time.Hour
)

Functions

func CheckWorkerModelRegister

func CheckWorkerModelRegister(ctx context.Context, h Interface, modelPath string) error

CheckWorkerModelRegister checks if a model has been registered, if not it raises an error on the API

func Create

func Create(ctx context.Context, h Interface) error

Create creates hatchery

func InitMetrics

func InitMetrics(ctx context.Context) error

func ModelInterpolateSecrets

func ModelInterpolateSecrets(hWithModels InterfaceWithModels, model *sdk.Model) error

ModelInterpolateSecrets interpolates secrets that can exists inside given model.

func NewWorkerToken

func NewWorkerToken(hatcheryName string, privateKey *rsa.PrivateKey, expiration time.Time, w SpawnArguments) (string, error)

NewWorkerToken .

func ParseArgs

func ParseArgs(str string) []string

ParseArgs splits str on spaces into a slice of strings taking into account any quoting (using ” or "") even inside args, and any backslash-escaping even without quotes:

`abc   def`       → ["abc", "def"]
` abc def `       → ["abc", "def"]
` '' "" `         → ["", ""]
` a'bc' d"e"f `   → ["abc", "def"]
`'a\bc\'' "def" ` → ["abc'", "def"]
` abc\ def `      → ["abc def"]

func ParseRequirementModel

func ParseRequirementModel(rm string) (string, map[string]string)

ParseRequirementModel parses a requirement model than returns the image name and the environment variables.

Example of input:

"postgres:latest env_1=blabla env_2=blabla env_3 env_4='zip'"

func SendSpawnInfo

func SendSpawnInfo(ctx context.Context, h Interface, jobID int64, spawnMsg sdk.SpawnMsg)

SendSpawnInfo sends a spawnInfo

func WorkerPool

func WorkerPool(ctx context.Context, h Interface, statusFilter ...string) ([]sdk.Worker, error)

WorkerPool returns all the worker owned by the hatchery h, registered or not on the CDS API

Types

type CacheNbAttemptsJobIDs

type CacheNbAttemptsJobIDs struct {
	// contains filtered or unexported fields
}

func (*CacheNbAttemptsJobIDs) Key

func (c *CacheNbAttemptsJobIDs) Key(id int64) string

func (*CacheNbAttemptsJobIDs) NewAttempt

func (c *CacheNbAttemptsJobIDs) NewAttempt(id int64) int

type Interface

type Interface interface {
	Name() string
	Type() string
	InitHatchery(ctx context.Context) error
	SpawnWorker(ctx context.Context, spawnArgs SpawnArguments) error
	CanSpawn(ctx context.Context, model *sdk.Model, jobID int64, requirements []sdk.Requirement) bool
	WorkersStarted(ctx context.Context) []string
	Service() *sdk.Service
	CDSClient() cdsclient.Interface
	Configuration() service.HatcheryCommonConfiguration
	Serve(ctx context.Context) error
	GetPrivateKey() *rsa.PrivateKey
	GetLogger() *logrus.Logger
	GetGoRoutines() *sdk.GoRoutines
}

Interface describe an interface for each hatchery mode Init create new clients for different api SpawnWorker creates a new vm instance CanSpawn return wether or not hatchery can spawn model WorkersStartedByModel returns the number of instances of given model started but not necessarily register on CDS yet WorkersStarted returns the number of instances started but not necessarily register on CDS yet Hatchery returns hatchery instance Client returns cdsclient instance ModelType returns type of hatchery NeedRegistration return true if worker model need regsitration ID returns hatchery id

type InterfaceWithModels

type InterfaceWithModels interface {
	Interface
	ModelType() string
	NeedRegistration(ctx context.Context, model *sdk.Model) bool
	WorkerModelsEnabled() ([]sdk.Model, error)
	WorkerModelSecretList(sdk.Model) (sdk.WorkerModelSecrets, error)
}

type JobIdentifiers

type JobIdentifiers struct {
	ServiceID  int64
	JobID      int64
	NodeRunID  int64
	RunID      int64
	WorkflowID int64
}

type Metrics

type Metrics struct {
	Jobs               *stats.Int64Measure
	JobsWebsocket      *stats.Int64Measure
	SpawnedWorkers     *stats.Int64Measure
	PendingWorkers     *stats.Int64Measure
	RegisteringWorkers *stats.Int64Measure
	CheckingWorkers    *stats.Int64Measure
	WaitingWorkers     *stats.Int64Measure
	BuildingWorkers    *stats.Int64Measure
	DisabledWorkers    *stats.Int64Measure
}

func GetMetrics

func GetMetrics() *Metrics

GetMetrics returns the metric stats measures

type SpawnArguments

type SpawnArguments struct {
	WorkerName   string `json:"worker_model"`
	WorkerToken  string
	Model        *sdk.Model        `json:"model"`
	JobName      string            `json:"job_name"`
	JobID        int64             `json:"job_id"`
	NodeRunID    int64             `json:"node_run_id"`
	NodeRunName  string            `json:"node_run_name"`
	Requirements []sdk.Requirement `json:"requirements"`
	RegisterOnly bool              `json:"register_only"`
	HatcheryName string            `json:"hatchery_name"`
	ProjectKey   string            `json:"project_key"`
	WorkflowName string            `json:"workflow_name"`
	WorkflowID   int64             `json:"workflow_id"`
	RunID        int64             `json:"run_id"`
}

SpawnArguments contains arguments to func SpawnWorker

func (*SpawnArguments) ModelName

func (s *SpawnArguments) ModelName() string

type WorkerJWTClaims

type WorkerJWTClaims struct {
	jwt.StandardClaims
	Worker SpawnArguments
}

WorkerJWTClaims is the specific claims format for Worker JWT