Documentation ¶
Index ¶
- Constants
- Variables
- func DimensionsFromTags(tags map[string][]string) []string
- func TagsForTask(name, id string, attempt int, rs RepoState, retryOf string, ...) []string
- type CacheRequest
- type CommitComment
- type CommitCommentSlice
- type Job
- type JobSlice
- type JobStatus
- type Machine
- type Patch
- type RepoComments
- type RepoState
- func (s *RepoState) Copy() RepoState
- func (s RepoState) GetCommit(repos repograph.Map) (*repograph.Commit, error)
- func (s RepoState) IsTryJob() bool
- func (s RepoState) Parents(repos repograph.Map) ([]RepoState, error)
- func (s RepoState) RowKey() string
- func (s RepoState) String() string
- func (s RepoState) Valid() bool
- type Task
- type TaskComment
- type TaskCommentSlice
- type TaskExecutor
- type TaskKey
- type TaskRequest
- type TaskResult
- type TaskSlice
- type TaskSpecComment
- type TaskSpecCommentSlice
- type TaskStatus
- type TaskSummary
Constants ¶
const ( ISSUE_SHORT_LENGTH = 2 BT_ROW_KEY_VERSION = "2" )
const ( // Swarming tags added by Task Scheduler. SWARMING_TAG_ATTEMPT = "sk_attempt" SWARMING_TAG_DIMENSION_PREFIX = "sk_dim_" SWARMING_TAG_FORCED_JOB_ID = "sk_forced_job_id" SWARMING_TAG_ID = "sk_id" SWARMING_TAG_ISSUE = "sk_issue" SWARMING_TAG_LUCI_PROJECT = "luci_project" SWARMING_TAG_MILO_HOST = "milo_host" SWARMING_TAG_NAME = "sk_name" SWARMING_TAG_PARENT_TASK_ID = "sk_parent_task_id" SWARMING_TAG_PATCHSET = "sk_patchset" SWARMING_TAG_REPO = "sk_repo" SWARMING_TAG_RETRY_OF = "sk_retry_of" SWARMING_TAG_REVISION = "sk_revision" SWARMING_TAG_SERVER = "sk_issue_server" // These two tags allow the swarming ui to point to the GoB repo SWARMING_TAG_SOURCE_REVISION = "source_revision" SWARMING_TAG_SOURCE_REPO = "source_repo" MILO_HOST = "https://ci.chromium.org/raw/build/%s" // Types of task executors. TaskExecutor_UseDefault = "" TaskExecutor_Swarming = "swarming" DefaultTaskExecutor = TaskExecutor_Swarming )
const DEFAULT_TEST_REPO = "go-on-now.git"
const ( // The path of the git cookie file used by task scheduler. go/gitauth will // create and manage this file. GitCookiesPath = "/tmp/.gitcookies" )
Variables ¶
var ( JOB_STATUS_BADNESS = map[JobStatus]int{ JOB_STATUS_SUCCESS: 0, JOB_STATUS_IN_PROGRESS: 1, JOB_STATUS_REQUESTED: 2, JOB_STATUS_CANCELED: 3, JOB_STATUS_FAILURE: 4, JOB_STATUS_MISHAP: 5, } VALID_JOB_STATUSES = []JobStatus{ JOB_STATUS_REQUESTED, JOB_STATUS_IN_PROGRESS, JOB_STATUS_SUCCESS, JOB_STATUS_FAILURE, JOB_STATUS_MISHAP, JOB_STATUS_CANCELED, } )
var (
ErrUnknownId = errors.New("Unknown ID")
)
var (
ValidTaskExecutors = []string{TaskExecutor_UseDefault, TaskExecutor_Swarming}
)
Functions ¶
func DimensionsFromTags ¶
DimensionsFromTags returns a set of dimensions based on the given tags.
Types ¶
type CacheRequest ¶
CacheRequest is a request for a named cache on a machine.
type CommitComment ¶
type CommitComment struct { Repo string `json:"repo"` Revision string `json:"revision"` // Timestamp is compared ignoring timezone. The timezone reflects User's // location. Timestamp time.Time `json:"time"` User string `json:"user,omitempty"` IgnoreFailure bool `json:"ignoreFailure"` Message string `json:"message,omitempty"` Deleted *bool `json:"deleted,omitempty"` }
CommitComment contains a comment about a commit. {Repo, Revision, Timestamp} is used as the unique id for this comment.
func MakeCommitComment ¶
MakeCommitComment creates a comment with its ID fields based on the given repo, commit, and ts, and other fields based on n.
func (CommitComment) Copy ¶
func (c CommitComment) Copy() *CommitComment
func (*CommitComment) Id ¶
func (c *CommitComment) Id() string
type CommitCommentSlice ¶
type CommitCommentSlice []*CommitComment
CommitCommentSlice implements sort.Interface. To sort commitComments []*CommitComment, use sort.Sort(CommitCommentSlice(commitComments)).
func (CommitCommentSlice) Len ¶
func (s CommitCommentSlice) Len() int
func (CommitCommentSlice) Less ¶
func (s CommitCommentSlice) Less(i, j int) bool
func (CommitCommentSlice) Swap ¶
func (s CommitCommentSlice) Swap(i, j int)
type Job ¶
type Job struct { // BuildbucketBuildId is the ID of the Buildbucket build with which this // Job is associated, if one exists. BuildbucketBuildId int64 `json:"buildbucketBuildId"` // BuildbucketLeaseKey is the lease key for running a Buildbucket build. // TODO(borenet): Maybe this doesn't belong in the DB. BuildbucketLeaseKey int64 `json:"buildbucketLeaseKey"` // BuildbucketPubSubTopic is the pub/sub topic to which we'll pub/sub // messages to update the build. BuildbucketPubSubTopic string `json:"buildbucketPubSubTopic"` // BuildbucketToken is used to authenticate requests to update the // Buildbucket build. BuildbucketToken string `json:"buildbucketToken"` // Created is the creation timestamp. This property should never change // for a given Job instance. Created time.Time `json:"created"` // DbModified is the time of the last successful call to JobDB.PutJob/s // for this Job, or zero if the job is new. DbModified time.Time `json:"dbModified"` // Dependencies maps out the DAG of TaskSpec names upon which this Job // depends. Keys are TaskSpec names and values are slices of TaskSpec // names indicating which TaskSpecs that TaskSpec depends on. This // property should never change for a given Job instance. Dependencies map[string][]string `json:"dependencies"` // Finished is the time at which all of the Job's dependencies finished, // successfully or not. Finished time.Time `json:"finished"` // Id is a unique identifier for the Job. This property should never // change for a given Job instance, after its initial insertion into the // DB. Id string `json:"id"` // IsForce indicates whether this is a manually-triggered Job, as // opposed to a normally scheduled one, or a try job. IsForce bool `json:"isForce"` // Name is a human-friendly descriptive name for the Job. All Jobs // generated from the same JobSpec have the same name. This property // should never change for a given Job instance. Name string `json:"name"` // Priority is an indicator of the relative priority of this Job. Priority float64 `json:"priority"` // RepoState is the current state of the repository for this Job. RepoState // Requested is the time at which this Job was requested. This is a // commit timestamp, tryjob request creation timestamp, time at which // the server received a force trigger job request, etc. Requested time.Time `json:"requested"` // Started is the timestamp at which the Job first entered // JOB_STATUS_IN_PROGRESS. Started time.Time `json:"started"` // Status is the current Job status, default JOB_STATUS_IN_PROGRESS. Status JobStatus `json:"status"` // StatusDetails provides additional details for the status of the Job, // including reasons it might have failed. This may be truncated due to // database storage limitations, so it should not include, for example, full // logs. StatusDetails string `json:"statusDetails"` // Tasks are the Task instances which satisfied the dependencies of // the Job. Keys are TaskSpec names and values are slices of TaskSummary // instances describing the Tasks. Tasks map[string][]*TaskSummary `json:"tasks"` }
Job represents a set of Tasks which are executed as part of a larger effort.
Job is stored as a GOB, so changes must maintain backwards compatibility. See gob package documentation for details, but generally:
- Ensure new fields can be initialized with their zero value.
- Do not change the type of any existing field.
- Leave removed fields commented out to ensure the field name is not reused.
- Add any new fields to the Copy() method.
func MakeFullJob ¶
MakeFullJob creates a Job instance which has all of its fields filled.
func MakeTestJob ¶
func (*Job) DeriveStatus ¶
DeriveStatus derives a JobStatus based on the TaskStatuses in the Job's dependency tree.
func (*Job) MakeTaskKey ¶
MakeTaskKey returns a TaskKey for the given Task name.
func (*Job) TraverseDependencies ¶
TraverseDependencies traces the dependency graph of the Job, calling the given function for each dependency. Only calls the function on task specs for whose dependencies the function has already been called. If the passed-in function returns an error, iteration stops and TraverseDependencies returns the same error.
type JobSlice ¶
type JobSlice []*Job
JobSlice implements sort.Interface. To sort jobs []*Job, use sort.Sort(JobSlice(jobs)).
type JobStatus ¶
type JobStatus string
JobStatus represents the current status of a Job. A JobStatus other than JOB_STATUS_IN_PROGRESS is final; we do not retry Jobs, only their component Tasks.
const ( // JOB_STATUS_REQUESTED indicates that we are aware of the Job but are not // yet ready to start running Tasks for it. JOB_STATUS_REQUESTED JobStatus = "REQUESTED" // JOB_STATUS_IN_PROGRESS indicates that one or more of the Job's // Task dependencies has not yet been satisfied. JOB_STATUS_IN_PROGRESS JobStatus = "" // JOB_STATUS_SUCCESS indicates that all of the Job's Task dependencies // completed successfully. JOB_STATUS_SUCCESS JobStatus = "SUCCESS" // JOB_STATUS_FAILURE indicates that one or more of the Job's Task // dependencies failed. JOB_STATUS_FAILURE JobStatus = "FAILURE" // JOB_STATUS_MISHAP indicates that one or more of the Job's Task // dependencies exited early with an error, died while in progress, was // manually canceled, expired while waiting on the queue, or timed out // before completing. JOB_STATUS_MISHAP JobStatus = "MISHAP" // JOB_STATUS_CANCELED indicates that the Job has been canceled. JOB_STATUS_CANCELED JobStatus = "CANCELED" // JOB_URL_TMPL is a template for Job URLs. JOB_URL_TMPL = "%s/job/%s" // DEFAULT_MAX_TASK_ATTEMPTS is the maximum number of attempts we'll // make of each TaskSpec in a Job. DEFAULT_MAX_TASK_ATTEMPTS = 2 )
func JobStatusFromTaskStatus ¶
func JobStatusFromTaskStatus(s TaskStatus) JobStatus
JobStatusFromTaskStatus returns a JobStatus based on a TaskStatus.
func WorseJobStatus ¶
WorseJobStatus returns the worse of the two JobStatus.
type Machine ¶
type Machine struct { ID string Dimensions []string IsDead bool IsQuarantined bool CurrentTaskID string }
Machine describes a machine which can run tasks.
type Patch ¶
type Patch struct { Issue string `json:"issue"` PatchRepo string `json:"patch_repo"` Patchset string `json:"patchset"` Server string `json:"server"` }
Patch describes a patch which may be applied to a code checkout.
func (Patch) GetPatchRef ¶
GetPatchRef returns the ref for the tryjob patch, if the RepoState includes a patch, and "" otherwise.
type RepoComments ¶
type RepoComments struct { // Repo is the repository (Repo field) of all the comments contained in // this RepoComments. Repo string // TaskComments maps commit hash and TaskSpec name to the comments for // the matching Task, sorted by timestamp. TaskComments map[string]map[string][]*TaskComment // TaskSpecComments maps TaskSpec name to the comments for that // TaskSpec, sorted by timestamp. TaskSpecComments map[string][]*TaskSpecComment // CommitComments maps commit hash to the comments for that commit, // sorted by timestamp. CommitComments map[string][]*CommitComment }
RepoComments contains comments that all pertain to the same repository.
func (*RepoComments) Copy ¶
func (orig *RepoComments) Copy() *RepoComments
type RepoState ¶
RepoState encapsulates all of the parameters which define the state of a repo.
func (RepoState) GetCommit ¶
GetCommit returns the repograph.Commit referenced by s, or an error if it can't be found.
func (RepoState) Parents ¶
Parents returns RepoStates referencing the "parents" of s. For try jobs, the parent is the base RepoState without a Patch. Otherwise, the parents reference the parent commits of s.Revision.
type Task ¶
type Task struct { // Attempt is the attempt number of this task, starting with zero. Attempt int `json:"attempt"` // Commits are the commits which were tested in this Task. The list may // change due to backfilling/bisecting. Commits []string `json:"commits"` // Created is the creation timestamp. Created time.Time `json:"created"` // DbModified is the time of the last successful call to TaskDB.PutTask/s for this // Task, or zero if the task is new. It is not related to the ModifiedTs time // of the associated Swarming task. DbModified time.Time `json:"dbModified"` // Finished is the time the task stopped running or expired from the queue, or // zero if the task is pending or running. Finished time.Time `json:"finished"` // Id is a generated unique identifier for this Task instance. Must be // URL-safe. Id string `json:"id"` // IsolatedOutput is the isolated hash of any outputs produced by this Task. // Filled in when the task is completed. This field will not be set if the // Task does not correspond to a Swarming task. IsolatedOutput string `json:"isolatedOutput"` // Jobs are the IDs of all Jobs which utilized this Task. Jobs []string `json:"jobs"` // MaxAttempts is the maximum number of attempts for this TaskSpec. MaxAttempts int `json:"max_attempts"` // ParentTaskIds are IDs of tasks which satisfied this task's dependencies. ParentTaskIds []string `json:"parentTaskIds"` // Properties contains key-value pairs from external sources. Both key and // value must be UTF-8 strings. Prefer a JavaScript identifier for key. Use // base64 encoding for binary data. Properties map[string]string `json:"properties"` // RetryOf is the ID of the task which this task is a retry of, if any. RetryOf string `json:"retryOf"` // Started is the time the task started running, or zero if the task is // pending, or the same as Finished if the task never ran. Started time.Time `json:"started"` // Status is the current task status, default TASK_STATUS_PENDING. Status TaskStatus `json:"status"` // SwarmingBotId is the ID of the Swarming bot that ran this task. This // field will not be set if the Task does not correspond to a Swarming task or // if the task is still pending. SwarmingBotId string `json:"swarmingBotId"` // SwarmingTaskId is the Swarming task ID. This field will not be set if the // Task does not correspond to a Swarming task. SwarmingTaskId string `json:"swarmingTaskId"` // TaskExecutor is the task executor used to run this Task. TaskExecutor string `json:"taskExecutor"` // TaskKey is a struct which describes aspects of the Task related // to the current state of the repo when it ran, and about the Task // itself. TaskKey }
Task describes a Swarming task generated from a TaskSpec, or a "fake" task that can not be executed on Swarming, but can be added to the DB and displayed as if it were a real TaskSpec.
Task is stored as a GOB, so changes must maintain backwards compatibility. See gob package documentation for details, but generally:
- Ensure new fields can be initialized with their zero value.
- Do not change the type of any existing field.
- Leave removed fields commented out to ensure the field name is not reused.
- Add any new fields to the Copy() method.
func (*Task) MakeTaskSummary ¶
func (t *Task) MakeTaskSummary() *TaskSummary
MakeTaskSummary creates a TaskSummary from the Task instance.
func (*Task) UpdateFromTaskResult ¶
func (orig *Task) UpdateFromTaskResult(res *TaskResult) (bool, error)
UpdateFromTaskResult sets or initializes t from data in s. If any changes were made to t, returns true.
Returns ErrUnknownId if the SwarmingTaskId does not match.
If empty, sets t.Id, t.Name, t.Repo, and t.Revision from s's tags named SWARMING_TAG_ID, SWARMING_TAG_NAME, SWARMING_TAG_REPO, and SWARMING_TAG_REVISION, and sets t.Created from s.CreatedTs. If these fields are non-empty, returns an error if they do not match.
Always sets t.Status, t.Started, t.Finished, and t.IsolatedOutput based on s.
type TaskComment ¶
type TaskComment struct { Repo string `json:"repo"` Revision string `json:"revision"` Name string `json:"name"` // Name of TaskSpec. // Timestamp is compared ignoring timezone. The timezone reflects User's // location. Timestamp time.Time `json:"time"` TaskId string `json:"taskId,omitempty"` User string `json:"user,omitempty"` Message string `json:"message,omitempty"` Deleted *bool `json:"deleted,omitempty"` }
TaskComment contains a comment about a Task. {Repo, Revision, Name, Timestamp} is used as the unique id for this comment. If TaskId is empty, the comment applies to all matching tasks.
func MakeTaskComment ¶
MakeTaskComment creates a comment with its ID fields based on the given repo, name, commit, and ts, and other fields based on n.
func (TaskComment) Copy ¶
func (c TaskComment) Copy() *TaskComment
func (*TaskComment) Id ¶
func (c *TaskComment) Id() string
type TaskCommentSlice ¶
type TaskCommentSlice []*TaskComment
TaskCommentSlice implements sort.Interface. To sort taskComments []*TaskComment, use sort.Sort(TaskCommentSlice(taskComments)).
func (TaskCommentSlice) Len ¶
func (s TaskCommentSlice) Len() int
func (TaskCommentSlice) Less ¶
func (s TaskCommentSlice) Less(i, j int) bool
func (TaskCommentSlice) Swap ¶
func (s TaskCommentSlice) Swap(i, j int)
type TaskExecutor ¶
type TaskExecutor interface { // GetFreeMachines returns all of the machines in the given pool which are // not currently running a task. GetFreeMachines(ctx context.Context, pool string) ([]*Machine, error) // GetPendingTasks returns all of the tasks in the given pool which have not // yet started. GetPendingTasks(ctx context.Context, pool string) ([]*TaskResult, error) // GetTaskResult retrieves the result of the given task. GetTaskResult(ctx context.Context, taskID string) (*TaskResult, error) // GetTaskCompletionStatuses returns a slice of bools indicating whether // each of the given tasks have finished. GetTaskCompletionStatuses(ctx context.Context, taskIDs []string) ([]bool, error) // TriggerTask triggers a new task to run. TriggerTask(ctx context.Context, req *TaskRequest) (*TaskResult, error) }
TaskExecutor is a framework for executing Tasks.
type TaskKey ¶
TaskKey is a struct used for identifying a Task instance. Note that more than one Task may have the same TaskKey, eg. in the case of retries.
func (TaskKey) IsForceRun ¶
IsForceRun indicates whether this Task is for a forced Job, which indicates that it shouldn't be de-duplicated.
type TaskRequest ¶
type TaskRequest struct { Caches []*CacheRequest CasInput string CipdPackages []*cipd.Package Command []string Dimensions []string Env map[string]string EnvPrefixes map[string][]string ExecutionTimeout time.Duration Expiration time.Duration Idempotent bool IoTimeout time.Duration Name string Outputs []string ServiceAccount string Tags []string TaskSchedulerTaskID string }
TaskRequest contains all of the information necessary to request execution of a Task.
type TaskResult ¶
type TaskResult struct { CasOutput string `json:"cas_output"` Created time.Time `json:"created"` Finished time.Time `json:"finished"` ID string `json:"id"` MachineID string `json:"machine_id"` Started time.Time `json:"started"` Status TaskStatus `json:"status"` Tags map[string][]string `json:"tags"` }
TaskResult describes a Task after it has been triggered.
Note that the JSON annotations are only used by machineserver to store this struct in CockroachDB as a JSONB field.
type TaskSlice ¶
type TaskSlice []*Task
TaskSlice implements sort.Interface. To sort tasks []*Task, use sort.Sort(TaskSlice(tasks)).
type TaskSpecComment ¶
type TaskSpecComment struct { Repo string `json:"repo"` Name string `json:"name"` // Name of TaskSpec. // Timestamp is compared ignoring timezone. The timezone reflects User's // location. Timestamp time.Time `json:"time"` User string `json:"user,omitempty"` Flaky bool `json:"flaky"` IgnoreFailure bool `json:"ignoreFailure"` Message string `json:"message,omitempty"` Deleted *bool `json:"deleted,omitempty"` }
TaskSpecComment contains a comment about a TaskSpec. {Repo, Name, Timestamp} is used as the unique id for this comment.
func MakeTaskSpecComment ¶
MakeTaskSpecComment creates a comment with its ID fields based on the given repo, name, and ts, and other fields based on n.
func (TaskSpecComment) Copy ¶
func (c TaskSpecComment) Copy() *TaskSpecComment
func (*TaskSpecComment) Id ¶
func (c *TaskSpecComment) Id() string
type TaskSpecCommentSlice ¶
type TaskSpecCommentSlice []*TaskSpecComment
TaskSpecCommentSlice implements sort.Interface. To sort taskSpecComments []*TaskSpecComment, use sort.Sort(TaskSpecCommentSlice(taskSpecComments)).
func (TaskSpecCommentSlice) Len ¶
func (s TaskSpecCommentSlice) Len() int
func (TaskSpecCommentSlice) Less ¶
func (s TaskSpecCommentSlice) Less(i, j int) bool
func (TaskSpecCommentSlice) Swap ¶
func (s TaskSpecCommentSlice) Swap(i, j int)
type TaskStatus ¶
type TaskStatus string
const ( // TASK_STATUS_PENDING indicates the task has not started. It is the empty // string so that it is the zero value of TaskStatus. TASK_STATUS_PENDING TaskStatus = "" // TASK_STATUS_RUNNING indicates the task is in progress. TASK_STATUS_RUNNING TaskStatus = "RUNNING" // TASK_STATUS_SUCCESS indicates the task completed successfully. TASK_STATUS_SUCCESS TaskStatus = "SUCCESS" // TASK_STATUS_FAILURE indicates the task completed with failures. TASK_STATUS_FAILURE TaskStatus = "FAILURE" // TASK_STATUS_MISHAP indicates the task exited early with an error, died // while in progress, was manually canceled, expired while waiting on the // queue, or timed out before completing. TASK_STATUS_MISHAP TaskStatus = "MISHAP" )
type TaskSummary ¶
type TaskSummary struct { Attempt int `json:"attempt"` Created time.Time `json:"created"` Id string `json:"id"` MaxAttempts int `json:"max_attempts"` Status TaskStatus `json:"status"` SwarmingTaskId string `json:"swarmingTaskId"` }
TaskSummary is a subset of the information found in a Task.
func (*TaskSummary) Copy ¶
func (t *TaskSummary) Copy() *TaskSummary
Copy returns a copy of the TaskSummary.