scheduling

Documentation

Index

• Constants
• Variables
• func ComputeBlamelist(ctx context.Context, cache cache.TaskCache, repo commitGetter, ...) ([]string, *types.Task, error)
• type BusyBotsDebugLog
• type TaskCandidate
  • func (c *TaskCandidate) AddJob(job *types.Job)
  • func (c *TaskCandidate) CopyNoDiagnostics() *TaskCandidate
  • func (s *TaskCandidate) GetDiagnostics() *taskCandidateDiagnostics
  • func (c *TaskCandidate) HasJob(job *types.Job) bool
  • func (c *TaskCandidate) MakeId() string
  • func (c *TaskCandidate) MakeTask() *types.Task
  • func (c *TaskCandidate) MakeTaskRequest(id, casInstance, pubSubTopic string) (*types.TaskRequest, error)
• type TaskCandidateSearchTerms
• type TaskScheduler
  • func NewTaskScheduler(ctx context.Context, d db.DB, bl *skip_tasks.DB, period time.Duration, ...) (*TaskScheduler, error)
  • func (s *TaskScheduler) CloneQueue() []*TaskCandidate
  • func (s *TaskScheduler) Close() error
  • func (s *TaskScheduler) GetSkipTasks() *skip_tasks.DB
  • func (s *TaskScheduler) HandleSwarmingPubSub(msg *swarming.PubSubTaskMessage) bool
  • func (s *TaskScheduler) MainLoop(ctx context.Context) error
  • func (s *TaskScheduler) QueueLen() int
  • func (s *TaskScheduler) SearchQueue(q *TaskCandidateSearchTerms) []*TaskCandidate
  • func (s *TaskScheduler) Start(ctx context.Context)
  • func (s *TaskScheduler) Status() *TaskSchedulerStatus
• type TaskSchedulerStatus

Constants

const (
	// Metric name for free bots.
	MEASUREMENT_FREE_BOT_COUNT = "free_bot_count"

	// FILTER_* are used as the value of the "filter" key in metrics; we record counts for all free
	// bots and all free bots after allocating pending tasks to bots.
	FILTER_ALL_FREE_BOTS       = "all_free_bots"
	FILTER_MINUS_PENDING_TASKS = "minus_pending_tasks"

	// Metric name for pending tasks.
	MEASUREMENT_PENDING_TASK_COUNT = "pending_swarming_task_count"
)

const (
	// Manually-forced jobs have high priority.
	CANDIDATE_SCORE_FORCE_RUN = 100.0

	// Try jobs have high priority, equal to building at HEAD when we're
	// 5 commits behind.
	CANDIDATE_SCORE_TRY_JOB = 10.0

	// When retrying a try job task that has failed, prioritize the retry
	// lower than tryjob tasks that haven't run yet.
	CANDIDATE_SCORE_TRY_JOB_RETRY_MULTIPLIER = 0.75

	// When bisecting or retrying a task that failed or had a mishap, add a bonus
	// to the raw score.
	//
	// A value of 0.75 means that a retry scores higher than a bisecting a
	// successful task with a blamelist of 2 commits, but lower than testing new
	// commits or bisecting successful tasks with blamelist of 3 or
	// more. Bisecting a failure with a blamelist of 2 commits scores the same as
	// bisecting a successful task with a blamelist of 4 commits.
	CANDIDATE_SCORE_FAILURE_OR_MISHAP_BONUS = 0.75

	// MAX_BLAMELIST_COMMITS is the maximum number of commits which are
	// allowed in a task blamelist before we stop tracing commit history.
	MAX_BLAMELIST_COMMITS = 500

	// Measurement name for task candidate counts by dimension set.
	MEASUREMENT_TASK_CANDIDATE_COUNT = "task_candidate_count"

	NUM_TOP_CANDIDATES = 50

	// To avoid errors resulting from DB transaction size limits, we
	// restrict the number of tasks triggered per TaskSpec (we insert tasks
	// into the DB in chunks by TaskSpec) to half of the DB transaction size
	// limit (since we may need to update an existing whose blamelist was
	// split by the new task).
	SCHEDULING_LIMIT_PER_TASK_SPEC = firestore.MAX_TRANSACTION_DOCS / 2

	GCS_MAIN_LOOP_DIAGNOSTICS_DIR = "MainLoop"
	GCS_DIAGNOSTICS_WRITE_TIMEOUT = 60 * time.Second
)

Variables

var (
	ERR_BLAMELIST_DONE = errors.New("ERR_BLAMELIST_DONE")
)

Functions

func ComputeBlamelist

func ComputeBlamelist(ctx context.Context, cache cache.TaskCache, repo commitGetter, taskName, repoName string, revision *repograph.Commit, commitsBuf []*repograph.Commit, tcg tasksCfgGetter, ct commitTester) ([]string, *types.Task, error)

ComputeBlamelist computes the blamelist for a new task, specified by name, repo, and revision. Returns the list of commits covered by the task, and any previous task which part or all of the blamelist was "stolen" from (see below). There are three cases:

1. The new task tests commits which have not yet been tested. Trace commit history, accumulating commits until we find commits which have been tested by previous tasks.

2. The new task runs at the same commit as a previous task. This is a retry, so the entire blamelist of the previous task is "stolen".

3. The new task runs at a commit which is in a previous task's blamelist, but no task has run at the same commit. This is a bisect. Trace commit history, "stealing" commits from the previous task until we find a commit which was covered by a *different* previous task.

Args:

• cache: TaskCache instance.
• repo: repograph.Graph instance corresponding to the repository of the task.
• taskName: Name of the task.
• repoName: Name of the repository for the task.
• revision: Revision at which the task would run.
• commitsBuf: Buffer for use as scratch space.

Types

type BusyBotsDebugLog

type BusyBotsDebugLog bool

const (
	BusyBotsDebugLoggingOn  BusyBotsDebugLog = true
	BusyBotsDebugLoggingOff BusyBotsDebugLog = false
)

type TaskCandidate

type TaskCandidate struct {
	Attempt int `json:"attempt"`
	// NB: Because multiple Jobs may share a Task, the BuildbucketBuildId
	// could be inherited from any matching Job. Therefore, this should be
	// used for non-critical, informational purposes only.
	BuildbucketBuildId int64    `json:"buildbucketBuildId"`
	Commits            []string `json:"commits"`
	CasInput           string   `json:"casInput"`
	CasDigests         []string `json:"casDigests"`
	// Jobs must be kept in sorted order; see AddJob.
	Jobs           []*types.Job `json:"jobs"`
	ParentTaskIds  []string     `json:"parentTaskIds"`
	RetryOf        string       `json:"retryOf"`
	Score          float64      `json:"score"`
	StealingFromId string       `json:"stealingFromId"`
	types.TaskKey
	TaskSpec    *specs.TaskSpec           `json:"taskSpec"`
	Diagnostics *taskCandidateDiagnostics `json:"diagnostics,omitempty"`
}

TaskCandidate is a struct used for determining which tasks to schedule.

func (*TaskCandidate) AddJob

func (c *TaskCandidate) AddJob(job *types.Job)

AddJob adds job to c.Jobs, unless already present.

func (*TaskCandidate) CopyNoDiagnostics

func (c *TaskCandidate) CopyNoDiagnostics() *TaskCandidate

CopyNoDiagnostics returns a copy of the taskCandidate, omitting the Diagnostics field.

func (*TaskCandidate) GetDiagnostics

func (s *TaskCandidate) GetDiagnostics() *taskCandidateDiagnostics

GetDiagnostics returns the taskCandidateDiagnostics for this taskCandidate, creating one if not present.

func (*TaskCandidate) HasJob

func (c *TaskCandidate) HasJob(job *types.Job) bool

HasJob returns true if job is a member of c.Jobs.

func (*TaskCandidate) MakeId

func (c *TaskCandidate) MakeId() string

MakeId generates a string ID for the taskCandidate.

func (*TaskCandidate) MakeTask

func (c *TaskCandidate) MakeTask() *types.Task

MakeTask instantiates a types.Task from the taskCandidate.

func (*TaskCandidate) MakeTaskRequest

func (c *TaskCandidate) MakeTaskRequest(id, casInstance, pubSubTopic string) (*types.TaskRequest, error)

MakeTaskRequest creates a SwarmingRpcsNewTaskRequest object from the taskCandidate.

type TaskCandidateSearchTerms

type TaskCandidateSearchTerms struct {
	types.TaskKey
	Dimensions []string `json:"dimensions"`
}

TaskCandidateSearchTerms includes fields used for searching task candidates.

type TaskScheduler

type TaskScheduler struct {
	// contains filtered or unexported fields
}

TaskScheduler is a struct used for scheduling tasks on bots.

func NewTaskScheduler

func NewTaskScheduler(ctx context.Context, d db.DB, bl *skip_tasks.DB, period time.Duration, numCommits int, repos repograph.Map, rbeCas cas.CAS, rbeCasInstance string, taskExecutors map[string]types.TaskExecutor, c *http.Client, timeDecayAmt24Hr float64, pools []string, pubsubTopic string, taskCfgCache task_cfg_cache.TaskCfgCache, ts oauth2.TokenSource, diagClient gcs.GCSClient, diagInstance string, debugBusyBots BusyBotsDebugLog) (*TaskScheduler, error)

func (*TaskScheduler) CloneQueue

func (s *TaskScheduler) CloneQueue() []*TaskCandidate

CloneQueue returns a full copy of the queue.

func (*TaskScheduler) Close

func (s *TaskScheduler) Close() error

Close cleans up resources used by the TaskScheduler.

func (*TaskScheduler) GetSkipTasks

func (s *TaskScheduler) GetSkipTasks() *skip_tasks.DB

func (*TaskScheduler) HandleSwarmingPubSub

func (s *TaskScheduler) HandleSwarmingPubSub(msg *swarming.PubSubTaskMessage) bool

HandleSwarmingPubSub loads the given Swarming task ID from Swarming and updates the associated types.Task in the database. Returns a bool indicating whether the pubsub message should be acknowledged.

func (*TaskScheduler) MainLoop

func (s *TaskScheduler) MainLoop(ctx context.Context) error

MainLoop runs a single end-to-end task scheduling loop.

func (*TaskScheduler) QueueLen

func (s *TaskScheduler) QueueLen() int

QueueLen returns the length of the queue.

func (*TaskScheduler) SearchQueue

func (s *TaskScheduler) SearchQueue(q *TaskCandidateSearchTerms) []*TaskCandidate

SearchQueue returns all task candidates in the queue which match the given TaskKey. Any blank fields are considered to be wildcards.

func (*TaskScheduler) Start

func (s *TaskScheduler) Start(ctx context.Context)

Start initiates the TaskScheduler's goroutines for scheduling tasks. beforeMainLoop will be run before each scheduling iteration.

func (*TaskScheduler) Status

func (s *TaskScheduler) Status() *TaskSchedulerStatus

Status returns the current status of the TaskScheduler.

type TaskSchedulerStatus

type TaskSchedulerStatus struct {
	LastScheduled time.Time        `json:"last_scheduled"`
	TopCandidates []*TaskCandidate `json:"top_candidates"`
}

TaskSchedulerStatus is a struct which provides status information about the TaskScheduler.