scheduler

Documentation

Index

• Constants
• func AlternateTaskFinder(distroID string) ([]task.Task, error)
• func GetExpectedDurations(runnableTasks []task.Task) (model.ProjectTaskDurations, error)
• func LegacyFindRunnableTasks(distroID string) ([]task.Task, error)
• func ParallelTaskFinder(distroID string) ([]task.Task, error)
• func PlanDistro(ctx context.Context, conf Configuration) error
• func RunnableTasksPipeline(distroID string) ([]task.Task, error)
• type CmpBasedTaskComparator
  • func NewCmpBasedTaskComparator() *CmpBasedTaskComparator
  • func (self *CmpBasedTaskComparator) Len() int
  • func (self *CmpBasedTaskComparator) Less(i, j int) bool
  • func (self *CmpBasedTaskComparator) Swap(i, j int)
• type CmpBasedTaskPrioritizer
  • func (prioritizer *CmpBasedTaskPrioritizer) PrioritizeTasks(distroId string, tasks []task.Task) ([]task.Task, error)
• type CmpBasedTaskQueues
• type Configuration
• type DBTaskQueuePersister
  • func (self *DBTaskQueuePersister) PersistTaskQueue(distro string, tasks []task.Task, taskDurations model.ProjectTaskDurations) ([]model.TaskQueueItem, error)
• type DeficitBasedHostAllocator
  • func (self *DeficitBasedHostAllocator) NewHostsNeeded(ctx context.Context, hostAllocatorData HostAllocatorData) (map[string]int, error)
• type DistroScheduleData
• type DurationBasedHostAllocator
  • func (self *DurationBasedHostAllocator) NewHostsNeeded(ctx context.Context, hostAllocatorData HostAllocatorData) (newHostsNeeded map[string]int, err error)
• type HostAllocator
• type HostAllocatorData
• type Runner
  • func (r *Runner) Name() string
  • func (r *Runner) Run(ctx context.Context, config *evergreen.Settings) error
• type ScheduledDistroTasksData
• type Scheduler
  • func (s *Scheduler) Schedule(ctx context.Context) error
• type TaskDurationEstimator
• type TaskFinder
  • func GetTaskFinder(name string) TaskFinder
• type TaskPrioritizer
• type TaskQueuePersister

Constants

const (
	// maximum turnaround we want to maintain for all hosts for a given distro
	MaxDurationPerDistroHost = 30 * time.Minute

	// for distro queues with tasks that appear on other queues, this constant
	// indicates the fraction of the total duration of shared tasks that we want
	// to account for when alternate distros are unable to satisfy the
	// turnaround requirement as determined by MaxDurationPerDistroHost
	SharedTasksAllocationProportion = 0.8
)

const (
	// builds per-distro queues of tasks for execution and creates host intents
	RunnerName = "scheduler"
)

Functions

func AlternateTaskFinder

func AlternateTaskFinder(distroID string) ([]task.Task, error)

func GetExpectedDurations

func GetExpectedDurations(runnableTasks []task.Task) (model.ProjectTaskDurations, error)

GetExpectedDurations returns the expected duration of tasks (by display name) on a project, buildvariant basis.

func LegacyFindRunnableTasks

func LegacyFindRunnableTasks(distroID string) ([]task.Task, error)

The old Task finderDBTaskFinder, with the dependency check implemented in Go, instead of using $graphLookup

func ParallelTaskFinder

func ParallelTaskFinder(distroID string) ([]task.Task, error)

func PlanDistro

func PlanDistro(ctx context.Context, conf Configuration) error

func RunnableTasksPipeline

func RunnableTasksPipeline(distroID string) ([]task.Task, error)

Types

type CmpBasedTaskComparator

type CmpBasedTaskComparator struct {
	// contains filtered or unexported fields
}

CmpBasedTaskComparator runs the tasks through a slice of comparator functions determining which is more important.

func NewCmpBasedTaskComparator

func NewCmpBasedTaskComparator() *CmpBasedTaskComparator

NewCmpBasedTaskComparator returns a new task prioritizer, using the default set of comparators as well as the setup functions necessary for those comparators.

func (*CmpBasedTaskComparator) Len

func (self *CmpBasedTaskComparator) Len() int

func (*CmpBasedTaskComparator) Less

func (self *CmpBasedTaskComparator) Less(i, j int) bool

func (*CmpBasedTaskComparator) Swap

func (self *CmpBasedTaskComparator) Swap(i, j int)

type CmpBasedTaskPrioritizer

type CmpBasedTaskPrioritizer struct{}

func (*CmpBasedTaskPrioritizer) PrioritizeTasks

func (prioritizer *CmpBasedTaskPrioritizer) PrioritizeTasks(distroId string, tasks []task.Task) ([]task.Task, error)

PrioritizeTask prioritizes the tasks to run. First splits the tasks into slices based on whether they are part of patch versions or automatically created versions. Then prioritizes each slice, and merges them. Returns a full slice of the prioritized tasks, and an error if one occurs.

type CmpBasedTaskQueues

type CmpBasedTaskQueues struct {
	HighPriorityTasks []task.Task
	PatchTasks        []task.Task
	RepotrackerTasks  []task.Task
}

CmpBasedTaskQueues represents the three types of queues that are created for merging together into one queue. The HighPriorityTasks list represent the tasks that are always placed at the front of the queue PatchTasks and RepotrackerTasks are interleaved after the high priority tasks.

type Configuration

type Configuration struct {
	DistroID   string
	TaskFinder string
}

type DBTaskQueuePersister

type DBTaskQueuePersister struct{}

DBTaskQueuePersister saves a queue to the database.

func (*DBTaskQueuePersister) PersistTaskQueue

func (self *DBTaskQueuePersister) PersistTaskQueue(distro string,
	tasks []task.Task,
	taskDurations model.ProjectTaskDurations) ([]model.TaskQueueItem, error)

PersistTaskQueue saves the task queue to the database. Returns an error if the db call returns an error.

type DeficitBasedHostAllocator

type DeficitBasedHostAllocator struct{}

DeficitBasedHostAllocator uses the difference between the number of free hosts and the number of tasks that need to be run as a metric for how many new hosts need to be spun up

func (*DeficitBasedHostAllocator) NewHostsNeeded

func (self *DeficitBasedHostAllocator) NewHostsNeeded(ctx context.Context,
	hostAllocatorData HostAllocatorData) (map[string]int, error)

NewHostsNeeded decides how many new hosts are needed for a distro by seeing if the number of tasks that need to be run for the distro is greater than the number of hosts currently free to run a task. Returns a map of distro-># of hosts to spawn.

type DistroScheduleData

type DistroScheduleData struct {
	// contains filtered or unexported fields
}

DistroScheduleData contains bookkeeping data that is used by distros to determine whether or not to allocate more hosts

type DurationBasedHostAllocator

type DurationBasedHostAllocator struct{}

DurationBasedHostAllocator computes the total time to completion of tasks running - per distro - and then uses that as a heuristic in determining how many new hosts to spin up

func (*DurationBasedHostAllocator) NewHostsNeeded

func (self *DurationBasedHostAllocator) NewHostsNeeded(ctx context.Context,
	hostAllocatorData HostAllocatorData) (newHostsNeeded map[string]int, err error)

NewHostsNeeded decides if new hosts are needed for a distro while taking the duration of running/scheduled tasks into consideration. Returns a map of distro to number of hosts to spawn.

type HostAllocator

type HostAllocator interface {
	NewHostsNeeded(context.Context, HostAllocatorData) (map[string]int, error)
}

HostAllocator is responsible for determining how many new hosts should be spun up. Parameters:

taskQueueItems: a map of distro name -> task queue items for that distro (a TaskQueue object)
distros: a map of distro name -> information on that distro (a model.Distro object)
existingDistroHosts: a map of distro name -> currently running hosts on that distro
projectTaskDurations: the expected duration of tasks by project and variant
taskRunDistros: a map of task id -> distros the task is allowed to run on

Returns a map of distro name -> how many hosts need to be spun up for that distro.

type HostAllocatorData

type HostAllocatorData struct {
	// contains filtered or unexported fields
}

HostAllocatorData is the set of parameters passed to a HostAllocator.

type Runner

type Runner struct{}

Runner runs the scheduler process.

func (*Runner) Name

func (r *Runner) Name() string

func (*Runner) Run

func (r *Runner) Run(ctx context.Context, config *evergreen.Settings) error

type ScheduledDistroTasksData

type ScheduledDistroTasksData struct {
	// contains filtered or unexported fields
}

ScheduledDistroTasksData contains data that is used to compute the expected duration of tasks within a distro's queue

type Scheduler

type Scheduler struct {
	*evergreen.Settings
	TaskPrioritizer
	TaskQueuePersister
	HostAllocator

	GetExpectedDurations TaskDurationEstimator
	FindRunnableTasks    TaskFinder
}

Responsible for prioritizing and scheduling tasks to be run, on a per-distro basis.

func (*Scheduler) Schedule

func (s *Scheduler) Schedule(ctx context.Context) error

Schedule all of the tasks to be run. Works by finding all of the tasks that are ready to be run, splitting them by distro, prioritizing them, and saving the per-distro queues. Then determines the number of new hosts to spin up for each distro, and spins them up.

type TaskDurationEstimator

type TaskDurationEstimator func([]task.Task) (model.ProjectTaskDurations, error)

TaskDurationEstimator is responsible for fetching the expected duration for a given set of runnable tasks.

type TaskFinder

type TaskFinder func(string) ([]task.Task, error)

func GetTaskFinder

func GetTaskFinder(name string) TaskFinder

type TaskPrioritizer

type TaskPrioritizer interface {
	// Takes in a slice of tasks and the current MCI settings.
	// Returns the slice of tasks, sorted in the order in which they should
	// be run, as well as an error if appropriate.
	PrioritizeTasks(distroId string, tasks []task.Task) ([]task.Task, error)
}

TaskPrioritizer is responsible for taking in a slice of tasks, and ordering them according to which should be run first.

type TaskQueuePersister

type TaskQueuePersister interface {
	PersistTaskQueue(distro string, tasks []task.Task,
		taskExpectedDuration model.ProjectTaskDurations) ([]model.TaskQueueItem,
		error)
}

TaskQueuePersister is responsible for taking a task queue for a particular distro and saving it.