scheduling

package
v0.0.0-...-4fe5f23 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 26, 2024 License: BSD-3-Clause Imports: 40 Imported by: 3

Documentation

Index

Constants

View Source 
const (
	// Metric name for free bots.
	MEASUREMENT_FREE_BOT_COUNT = "free_bot_count"

	// FILTER_* are used as the value of the "filter" key in metrics; we record counts for all free
	// bots and all free bots after allocating pending tasks to bots.
	FILTER_ALL_FREE_BOTS       = "all_free_bots"
	FILTER_MINUS_PENDING_TASKS = "minus_pending_tasks"

	// Metric name for pending tasks.
	MEASUREMENT_PENDING_TASK_COUNT = "pending_swarming_task_count"
)
View Source 
const (
	// Manually-forced jobs have high priority.
	CANDIDATE_SCORE_FORCE_RUN = 100.0

	// Try jobs have high priority, equal to building at HEAD when we're
	// 5 commits behind.
	CANDIDATE_SCORE_TRY_JOB = 10.0

	// When retrying a try job task that has failed, prioritize the retry
	// lower than tryjob tasks that haven't run yet.
	CANDIDATE_SCORE_TRY_JOB_RETRY_MULTIPLIER = 0.75

	// When bisecting or retrying a task that failed or had a mishap, add a bonus
	// to the raw score.
	//
	// A value of 0.75 means that a retry scores higher than a bisecting a
	// successful task with a blamelist of 2 commits, but lower than testing new
	// commits or bisecting successful tasks with blamelist of 3 or
	// more. Bisecting a failure with a blamelist of 2 commits scores the same as
	// bisecting a successful task with a blamelist of 4 commits.
	CANDIDATE_SCORE_FAILURE_OR_MISHAP_BONUS = 0.75

	// MAX_BLAMELIST_COMMITS is the maximum number of commits which are
	// allowed in a task blamelist before we stop tracing commit history.
	MAX_BLAMELIST_COMMITS = 500

	// Measurement name for task candidate counts by dimension set.
	MEASUREMENT_TASK_CANDIDATE_COUNT = "task_candidate_count"

	NUM_TOP_CANDIDATES = 50

	// To avoid errors resulting from DB transaction size limits, we
	// restrict the number of tasks triggered per TaskSpec (we insert tasks
	// into the DB in chunks by TaskSpec) to half of the DB transaction size
	// limit (since we may need to update an existing whose blamelist was
	// split by the new task).
	SCHEDULING_LIMIT_PER_TASK_SPEC = firestore.MAX_TRANSACTION_DOCS / 2

	GCS_MAIN_LOOP_DIAGNOSTICS_DIR = "MainLoop"
	GCS_DIAGNOSTICS_WRITE_TIMEOUT = 60 * time.Second
)

Variables

View Source 
var (
	ERR_BLAMELIST_DONE = errors.New("ERR_BLAMELIST_DONE")
)

Functions

func ComputeBlamelist 

func ComputeBlamelist(ctx context.Context, cache cache.TaskCache, repo commitGetter, taskName, repoName string, revision *repograph.Commit, commitsBuf []*repograph.Commit, tcg tasksCfgGetter, ct commitTester) ([]string, *types.Task, error)

ComputeBlamelist computes the blamelist for a new task, specified by name, repo, and revision. Returns the list of commits covered by the task, and any previous task which part or all of the blamelist was "stolen" from (see below). There are three cases:

  1. The new task tests commits which have not yet been tested. Trace commit history, accumulating commits until we find commits which have been tested by previous tasks.

  2. The new task runs at the same commit as a previous task. This is a retry, so the entire blamelist of the previous task is "stolen".

  3. The new task runs at a commit which is in a previous task's blamelist, but no task has run at the same commit. This is a bisect. Trace commit history, "stealing" commits from the previous task until we find a commit which was covered by a *different* previous task.

Args:

  • cache: TaskCache instance.
  • repo: repograph.Graph instance corresponding to the repository of the task.
  • taskName: Name of the task.
  • repoName: Name of the repository for the task.
  • revision: Revision at which the task would run.
  • commitsBuf: Buffer for use as scratch space.

Types

type BusyBotsDebugLog 

type BusyBotsDebugLog bool
const (
	BusyBotsDebugLoggingOn  BusyBotsDebugLog = true
	BusyBotsDebugLoggingOff BusyBotsDebugLog = false
)

type TaskCandidate 

type TaskCandidate struct {
	Attempt int `json:"attempt"`
	// NB: Because multiple Jobs may share a Task, the BuildbucketBuildId
	// could be inherited from any matching Job. Therefore, this should be
	// used for non-critical, informational purposes only.
	BuildbucketBuildId int64    `json:"buildbucketBuildId"`
	Commits            []string `json:"commits"`
	CasInput           string   `json:"casInput"`
	CasDigests         []string `json:"casDigests"`
	// Jobs must be kept in sorted order; see AddJob.
	Jobs           []*types.Job `json:"jobs"`
	ParentTaskIds  []string     `json:"parentTaskIds"`
	RetryOf        string       `json:"retryOf"`
	Score          float64      `json:"score"`
	StealingFromId string       `json:"stealingFromId"`
	types.TaskKey
	TaskSpec    *specs.TaskSpec           `json:"taskSpec"`
	Diagnostics *taskCandidateDiagnostics `json:"diagnostics,omitempty"`
}

TaskCandidate is a struct used for determining which tasks to schedule.

func (*TaskCandidate) AddJob 

func (c *TaskCandidate) AddJob(job *types.Job)

AddJob adds job to c.Jobs, unless already present.

func (*TaskCandidate) CopyNoDiagnostics 

func (c *TaskCandidate) CopyNoDiagnostics() *TaskCandidate

CopyNoDiagnostics returns a copy of the taskCandidate, omitting the Diagnostics field.

func (*TaskCandidate) GetDiagnostics 

func (s *TaskCandidate) GetDiagnostics() *taskCandidateDiagnostics

GetDiagnostics returns the taskCandidateDiagnostics for this taskCandidate, creating one if not present.

func (*TaskCandidate) HasJob 

func (c *TaskCandidate) HasJob(job *types.Job) bool

HasJob returns true if job is a member of c.Jobs.

func (*TaskCandidate) MakeId 

func (c *TaskCandidate) MakeId() string

MakeId generates a string ID for the taskCandidate.

func (*TaskCandidate) MakeTask 

func (c *TaskCandidate) MakeTask() *types.Task

MakeTask instantiates a types.Task from the taskCandidate.

func (*TaskCandidate) MakeTaskRequest 

func (c *TaskCandidate) MakeTaskRequest(id, casInstance, pubSubTopic string) (*types.TaskRequest, error)

MakeTaskRequest creates a SwarmingRpcsNewTaskRequest object from the taskCandidate.

type TaskCandidateSearchTerms 

type TaskCandidateSearchTerms struct {
	types.TaskKey
	Dimensions []string `json:"dimensions"`
}

TaskCandidateSearchTerms includes fields used for searching task candidates.

type TaskScheduler 

type TaskScheduler struct {
	// contains filtered or unexported fields
}

TaskScheduler is a struct used for scheduling tasks on bots.

func NewTaskScheduler 

func NewTaskScheduler(ctx context.Context, d db.DB, bl *skip_tasks.DB, period time.Duration, numCommits int, repos repograph.Map, rbeCas cas.CAS, rbeCasInstance string, taskExecutors map[string]types.TaskExecutor, c *http.Client, timeDecayAmt24Hr float64, pools []string, pubsubTopic string, taskCfgCache task_cfg_cache.TaskCfgCache, ts oauth2.TokenSource, diagClient gcs.GCSClient, diagInstance string, debugBusyBots BusyBotsDebugLog) (*TaskScheduler, error)

func (*TaskScheduler) CloneQueue 

func (s *TaskScheduler) CloneQueue() []*TaskCandidate

CloneQueue returns a full copy of the queue.

func (*TaskScheduler) Close 

func (s *TaskScheduler) Close() error

Close cleans up resources used by the TaskScheduler.

func (*TaskScheduler) GetSkipTasks 

func (s *TaskScheduler) GetSkipTasks() *skip_tasks.DB

func (*TaskScheduler) HandleSwarmingPubSub 

func (s *TaskScheduler) HandleSwarmingPubSub(msg *swarming.PubSubTaskMessage) bool

HandleSwarmingPubSub loads the given Swarming task ID from Swarming and updates the associated types.Task in the database. Returns a bool indicating whether the pubsub message should be acknowledged.

func (*TaskScheduler) MainLoop 

func (s *TaskScheduler) MainLoop(ctx context.Context) error

MainLoop runs a single end-to-end task scheduling loop.

func (*TaskScheduler) QueueLen 

func (s *TaskScheduler) QueueLen() int

QueueLen returns the length of the queue.

func (*TaskScheduler) SearchQueue 

func (s *TaskScheduler) SearchQueue(q *TaskCandidateSearchTerms) []*TaskCandidate

SearchQueue returns all task candidates in the queue which match the given TaskKey. Any blank fields are considered to be wildcards.

func (*TaskScheduler) Start 

func (s *TaskScheduler) Start(ctx context.Context)

Start initiates the TaskScheduler's goroutines for scheduling tasks. beforeMainLoop will be run before each scheduling iteration.

func (*TaskScheduler) Status 

func (s *TaskScheduler) Status() *TaskSchedulerStatus

Status returns the current status of the TaskScheduler.

type TaskSchedulerStatus 

type TaskSchedulerStatus struct {
	LastScheduled time.Time        `json:"last_scheduled"`
	TopCandidates []*TaskCandidate `json:"top_candidates"`
}

TaskSchedulerStatus is a struct which provides status information about the TaskScheduler.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.