specs

package
v0.0.0-...-03d6fc4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 23, 2019 License: BSD-3-Clause Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DEFAULT_TASK_SPEC_MAX_ATTEMPTS = types.DEFAULT_MAX_TASK_ATTEMPTS
	DEFAULT_NUM_WORKERS            = 10

	// The default JobSpec.Priority, when unspecified or invalid.
	DEFAULT_JOB_SPEC_PRIORITY = 0.5

	TASKS_CFG_FILE = "infra/bots/tasks.json"

	// By default, all jobs trigger on any branch for which they are
	// defined.
	TRIGGER_ANY_BRANCH = ""
	// Run this job on the master branch only, even if it is defined on
	// others.
	TRIGGER_MASTER_ONLY = "master"
	// Trigger this job every night.
	TRIGGER_NIGHTLY = periodic.TRIGGER_NIGHTLY
	// Don't trigger this job automatically. It will only be run when
	// explicitly triggered via a try job or a force trigger.
	TRIGGER_ON_DEMAND = "on demand"
	// Trigger this job weekly.
	TRIGGER_WEEKLY = periodic.TRIGGER_WEEKLY

	VARIABLE_SYNTAX = "<(%s)"

	VARIABLE_BUILDBUCKET_BUILD_ID = "BUILDBUCKET_BUILD_ID"
	VARIABLE_CODEREVIEW_SERVER    = "CODEREVIEW_SERVER"
	VARIABLE_ISSUE                = "ISSUE"
	VARIABLE_ISSUE_SHORT          = "ISSUE_SHORT"
	VARIABLE_PATCH_REF            = "PATCH_REF"
	VARIABLE_PATCH_REPO           = "PATCH_REPO"
	VARIABLE_PATCH_STORAGE        = "PATCH_STORAGE"
	VARIABLE_PATCHSET             = "PATCHSET"
	VARIABLE_REPO                 = "REPO"
	VARIABLE_REVISION             = "REVISION"
	VARIABLE_TASK_ID              = "TASK_ID"
	VARIABLE_TASK_NAME            = "TASK_NAME"

	// BigTable used for storing TaskCfgs.
	BT_INSTANCE_PROD     = "tasks-cfg-prod"
	BT_INSTANCE_INTERNAL = "tasks-cfg-internal"
	BT_INSTANCE_STAGING  = "tasks-cfg-staging"

	// We use a single BigTable table for storing gob-encoded TaskSpecs and
	// JobSpecs.
	BT_TABLE = "tasks-cfg"

	// We use a single BigTable column family.
	BT_COLUMN_FAMILY = "CFGS"

	// We use a single BigTable column which stores gob-encoded TaskSpecs
	// and JobSpecs.
	BT_COLUMN = "CFG"

	INSERT_TIMEOUT = 30 * time.Second
	QUERY_TIMEOUT  = 5 * time.Second
)

Variables

View Source 
var (
	// Fully-qualified BigTable column name.
	BT_COLUMN_FULL = fmt.Sprintf("%s:%s", BT_COLUMN_FAMILY, BT_COLUMN)

	PLACEHOLDER_BUILDBUCKET_BUILD_ID = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_BUILDBUCKET_BUILD_ID)
	PLACEHOLDER_CODEREVIEW_SERVER    = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_CODEREVIEW_SERVER)
	PLACEHOLDER_ISSUE                = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_ISSUE)
	PLACEHOLDER_ISSUE_SHORT          = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_ISSUE_SHORT)
	PLACEHOLDER_PATCH_REF            = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCH_REF)
	PLACEHOLDER_PATCH_REPO           = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCH_REPO)
	PLACEHOLDER_PATCH_STORAGE        = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCH_STORAGE)
	PLACEHOLDER_PATCHSET             = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCHSET)
	PLACEHOLDER_REPO                 = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_REPO)
	PLACEHOLDER_REVISION             = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_REVISION)
	PLACEHOLDER_TASK_ID              = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_TASK_ID)
	PLACEHOLDER_TASK_NAME            = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_TASK_NAME)
	PLACEHOLDER_ISOLATED_OUTDIR      = "${ISOLATED_OUTDIR}"

	PERIODIC_TRIGGERS = []string{TRIGGER_NIGHTLY, TRIGGER_WEEKLY}
)

Functions

func EncodeTasksCfg 

func EncodeTasksCfg(cfg *TasksCfg) ([]byte, error)

EncoderTasksCfg writes the TasksCfg to a byte slice.

func GetCheckoutRoot 

func GetCheckoutRoot() (string, error)

GetCheckoutRoot returns the path of the root of the checkout.

func WriteTasksCfg 

func WriteTasksCfg(cfg *TasksCfg, repoDir string) error

WriteTasksCfg writes the task cfg to the given repo.

func WriteTasksCfgToBigTable 

func WriteTasksCfgToBigTable(table *bigtable.Table, rs types.RepoState, cfg *TasksCfg, err error) error

Types

type Cache 

type Cache struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

Cache is a struct representing a named cache which is used by a task.

type CipdPackage 

type CipdPackage struct {
	Name    string `json:"name"`
	Path    string `json:"path"`
	Version string `json:"version"`
}

CipdPackage is a struct representing a CIPD package which needs to be installed on a bot for a particular task.

type JobSpec 

type JobSpec struct {
	// Priority indicates the relative priority of the job, with 0 < p <= 1,
	// where higher values result in scheduling the job's tasks sooner. If
	// unspecified or outside this range, DEFAULT_JOB_SPEC_PRIORITY is used.
	// Each task derives its priority from the set of jobs that depend upon
	// it. A task's priority is
	//   1 - (1-<job1 priority>)(1-<job2 priority>)...(1-<jobN priority>)
	// A task at HEAD with a priority of 1 and a blamelist of 1 commit has
	// approximately the same score as a task at HEAD with a priority of 0.57
	// and a blamelist of 2 commits.
	// A backfill task with a priority of 1 that bisects a blamelist of 2
	// commits has the same score as another backfill task at the same
	// commit with a priority of 0.4 that bisects a blamelist of 4 commits.
	Priority float64 `json:"priority,omitempty"`
	// The names of TaskSpecs that are direct dependencies of this JobSpec.
	TaskSpecs []string `json:"tasks"`
	// One of the TRIGGER_* constants; see documentation above.
	Trigger string `json:"trigger,omitempty"`
}

JobSpec is a struct which describes a set of TaskSpecs to run as part of a larger effort.

func (*JobSpec) Copy 

func (j *JobSpec) Copy() *JobSpec

Copy returns a copy of the JobSpec.

func (*JobSpec) GetTaskSpecDAG 

func (j *JobSpec) GetTaskSpecDAG(cfg *TasksCfg) (map[string][]string, error)

GetTaskSpecDAG returns a map describing all of the dependencies of the JobSpec. Its keys are TaskSpec names and values are TaskSpec names upon which the keys depend.

type TaskCfgCache 

type TaskCfgCache struct {
	// contains filtered or unexported fields
}

TaskCfgCache is a struct used for caching tasks cfg files. The user should periodically call Cleanup() to remove old entries.

func NewTaskCfgCache 

func NewTaskCfgCache(ctx context.Context, repos repograph.Map, depotToolsDir, workdir string, numWorkers int, btProject, btInstance string, ts oauth2.TokenSource) (*TaskCfgCache, error)

NewTaskCfgCache returns a TaskCfgCache instance.

func (*TaskCfgCache) Cleanup 

func (c *TaskCfgCache) Cleanup(period time.Duration) error

Cleanup removes cache entries which are outside of our scheduling window.

func (*TaskCfgCache) Close 

func (c *TaskCfgCache) Close() error

Close frees up resources used by the TaskCfgCache.

func (*TaskCfgCache) GetAddedTaskSpecsForRepoStates 

func (c *TaskCfgCache) GetAddedTaskSpecsForRepoStates(ctx context.Context, rss []types.RepoState) (map[types.RepoState]util.StringSet, error)

GetAddedTaskSpecsForRepoStates returns a mapping from each input RepoState to the set of task names that were added at that RepoState.

func (*TaskCfgCache) GetJobSpec 

func (c *TaskCfgCache) GetJobSpec(ctx context.Context, rs types.RepoState, name string) (*JobSpec, error)

GetJobSpec returns the JobSpec at the given RepoState, or an error if no such JobSpec exists.

func (*TaskCfgCache) GetTaskSpec 

func (c *TaskCfgCache) GetTaskSpec(ctx context.Context, rs types.RepoState, name string) (*TaskSpec, error)

GetTaskSpec returns the TaskSpec at the given RepoState, or an error if no such TaskSpec exists.

func (*TaskCfgCache) GetTaskSpecsForRepoStates 

func (c *TaskCfgCache) GetTaskSpecsForRepoStates(ctx context.Context, rs []types.RepoState) (map[types.RepoState]map[string]*TaskSpec, error)

GetTaskSpecsForRepoStates returns a set of TaskSpecs for each of the given set of RepoStates, keyed by RepoState and TaskSpec name.

func (*TaskCfgCache) MakeJob 

func (c *TaskCfgCache) MakeJob(ctx context.Context, rs types.RepoState, name string) (*types.Job, error)

MakeJob is a helper function which retrieves the given JobSpec at the given RepoState and uses it to create a Job instance.

func (*TaskCfgCache) ReadTasksCfg 

func (c *TaskCfgCache) ReadTasksCfg(ctx context.Context, rs types.RepoState) (*TasksCfg, error)

ReadTasksCfg reads the task cfg file from the given RepoState and returns it. Stores a cache of already-read task cfg files. Syncs the repo and reads the file if needed.

func (*TaskCfgCache) RecentSpecsAndCommits 

func (c *TaskCfgCache) RecentSpecsAndCommits() ([]string, []string, []string)

RecentSpecsAndCommits returns lists of recent job and task spec names and commit hashes.

func (*TaskCfgCache) TempGitRepo 

func (c *TaskCfgCache) TempGitRepo(ctx context.Context, rs types.RepoState, botUpdate bool, fn func(*git.TempCheckout) error) error

TempGitRepo creates a git repository in a temporary directory, gets it into the given RepoState, and runs the given function inside the repo dir.

This method uses a worker pool; if all workers are busy, it will block until one is free.

type TaskSpec 

type TaskSpec struct {
	// Caches are named Swarming caches which should be used for this task.
	Caches []*Cache `json:"caches,omitempty"`

	// CipdPackages are CIPD packages which should be installed for the task.
	CipdPackages []*CipdPackage `json:"cipd_packages,omitempty"`

	// Command is the command to run in the Swarming task. If not specified
	// here, it should be specified in the .isolate file.
	Command []string `json:"command,omitempty"`

	// Dependencies are names of other TaskSpecs for tasks which need to run
	// before this task.
	Dependencies []string `json:"dependencies,omitempty"`

	// Dimensions are Swarming bot dimensions which describe the type of bot
	// which may run this task.
	Dimensions []string `json:"dimensions"`

	// Environment is a set of environment variables needed by the task.
	Environment map[string]string `json:"environment,omitempty"`

	// EnvPrefixes are prefixes to add to environment variables for the task,
	// for example, adding directories to PATH. Keys are environment variable
	// names and values are multiple values to add for the variable.
	EnvPrefixes map[string][]string `json:"env_prefixes,omitempty"`

	// ExecutionTimeout is the maximum amount of time the task is allowed
	// to take.
	ExecutionTimeout time.Duration `json:"execution_timeout_ns,omitempty"`

	// Expiration is how long the task may remain in the pending state
	// before it is abandoned.
	Expiration time.Duration `json:"expiration_ns,omitempty"`

	// ExtraArgs are extra command-line arguments to pass to the task.
	ExtraArgs []string `json:"extra_args,omitempty"`

	// ExtraTags are extra tags to add to the Swarming task.
	ExtraTags map[string]string `json:"extra_tags,omitempty"`

	// IoTimeout is the maximum amount of time which the task may take to
	// communicate with the server.
	IoTimeout time.Duration `json:"io_timeout_ns,omitempty"`

	// Isolate is the name of the isolate file used by this task.
	Isolate string `json:"isolate"`

	// MaxAttempts is the maximum number of attempts for this TaskSpec. If
	// zero, DEFAULT_TASK_SPEC_MAX_ATTEMPTS is used.
	MaxAttempts int `json:"max_attempts,omitempty"`

	// Outputs are files and/or directories to use as outputs for the task.
	// Paths are relative to the task workdir. No error occurs if any of
	// these is missing.
	Outputs []string `json:"outputs,omitempty"`

	// This field is ignored.
	Priority float64 `json:"priority,omitempty"`

	// ServiceAccount indicates the Swarming service account to use for the
	// task. If not specified, we will attempt to choose a suitable default.
	ServiceAccount string `json:"service_account,omitempty"`
}

TaskSpec is a struct which describes a Swarming task to run. Be sure to add any new fields to the Copy() method.

func (*TaskSpec) Copy 

func (t *TaskSpec) Copy() *TaskSpec

Copy returns a copy of the TaskSpec.

func (*TaskSpec) Validate 

func (t *TaskSpec) Validate(cfg *TasksCfg) error

Validate ensures that the TaskSpec is defined properly.

type TasksCfg 

type TasksCfg struct {
	// Jobs is a map whose keys are JobSpec names and values are JobSpecs
	// which describe sets of tasks to run.
	Jobs map[string]*JobSpec `json:"jobs"`

	// Tasks is a map whose keys are TaskSpec names and values are TaskSpecs
	// detailing the Swarming tasks which may be run.
	Tasks map[string]*TaskSpec `json:"tasks"`
}

TasksCfg is a struct which describes all Swarming tasks for a repo at a particular commit.

func GetTasksCfgFromBigTable 

func GetTasksCfgFromBigTable(table *bigtable.Table, rs types.RepoState) (*TasksCfg, error)

func ParseTasksCfg 

func ParseTasksCfg(contents string) (*TasksCfg, error)

ParseTasksCfg parses the given task cfg file contents and returns the config.

func ReadTasksCfg 

func ReadTasksCfg(repoDir string) (*TasksCfg, error)

ReadTasksCfg reads the task cfg file from the given dir and returns it.

func (*TasksCfg) Validate 

func (c *TasksCfg) Validate() error

Validate returns an error if the TasksCfg is not valid.

type TasksCfgBuilder 

type TasksCfgBuilder struct {
	// contains filtered or unexported fields
}

TasksCfgBuilder is a helper struct used for building a TasksCfg.

func MustNewTasksCfgBuilder 

func MustNewTasksCfgBuilder() *TasksCfgBuilder

MustNewTasksCfgBuilder returns a TasksCfgBuilder instance. Panics on error.

func NewTasksCfgBuilder 

func NewTasksCfgBuilder() (*TasksCfgBuilder, error)

NewTasksCfgBuilder returns a TasksCfgBuilder instance.

func (*TasksCfgBuilder) AddJob 

func (b *TasksCfgBuilder) AddJob(name string, j *JobSpec) error

AddJob adds a JobSpec to the TasksCfgBuilder.

func (*TasksCfgBuilder) AddTask 

func (b *TasksCfgBuilder) AddTask(name string, t *TaskSpec) error

AddTask adds a TaskSpec to the TasksCfgBuilder. Returns an error if the config already contains a Task with the same name and a different implementation.

func (*TasksCfgBuilder) CheckoutRoot 

func (b *TasksCfgBuilder) CheckoutRoot() string

CheckoutRoot returns the path to the root of the client checkout.

func (*TasksCfgBuilder) Finish 

func (b *TasksCfgBuilder) Finish() error

Finish validates and writes out the TasksCfg, or, if the --test flag is provided, verifies that the contents have not changed.

func (*TasksCfgBuilder) GetCipdPackageFromAsset 

func (b *TasksCfgBuilder) GetCipdPackageFromAsset(assetName string) (*CipdPackage, error)

GetCipdPackageFromAsset reads the version information for the given asset and returns a CipdPackage instance.

func (*TasksCfgBuilder) MustAddJob 

func (b *TasksCfgBuilder) MustAddJob(name string, j *JobSpec)

MustAddJob adds a JobSpec to the TasksCfgBuilder and panics on failure.

func (*TasksCfgBuilder) MustAddTask 

func (b *TasksCfgBuilder) MustAddTask(name string, t *TaskSpec)

MustAddTask adds a TaskSpec to the TasksCfgBuilder and panics on failure.

func (*TasksCfgBuilder) MustFinish 

func (b *TasksCfgBuilder) MustFinish()

MustFinish validates and writes out the TasksCfg, or, if the --test flag is provided, verifies that the contents have not changed. Panics on failure.

func (*TasksCfgBuilder) MustGetCipdPackageFromAsset 

func (b *TasksCfgBuilder) MustGetCipdPackageFromAsset(assetName string) *CipdPackage

MustGetCipdPackageFromAsset reads the version information for the given asset and returns a CipdPackage instance. Panics on failure.

func (*TasksCfgBuilder) SetAssetsDir 

func (b *TasksCfgBuilder) SetAssetsDir(assetsDir string)

SetAssetsDir sets the directory path used for assets.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.