specs

package
v0.0.0-...-5fa633d Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 13, 2024 License: BSD-3-Clause Imports: 20 Imported by: 5

Documentation

Index

Constants

View Source 
const (
	DEFAULT_TASK_SPEC_MAX_ATTEMPTS = types.DEFAULT_MAX_TASK_ATTEMPTS

	// The default JobSpec.Priority, when unspecified or invalid.
	DEFAULT_JOB_SPEC_PRIORITY = 0.5

	TASKS_CFG_FILE = "infra/bots/tasks.json"

	// By default, all jobs trigger on any branch for which they are
	// defined.
	TRIGGER_ANY_BRANCH = ""
	// Run this job on the main branch only, even if it is defined on others.
	TRIGGER_MASTER_ONLY = "master"
	TRIGGER_MAIN_ONLY   = "main"
	// Trigger this job every night.
	TRIGGER_NIGHTLY = periodic.TRIGGER_NIGHTLY
	// Don't trigger this job automatically. It will only be run when
	// explicitly triggered via a try job or a force trigger.
	TRIGGER_ON_DEMAND = "on demand"
	// Trigger this job weekly.
	TRIGGER_WEEKLY = periodic.TRIGGER_WEEKLY

	VARIABLE_SYNTAX = "<(%s)"

	VARIABLE_BUILDBUCKET_BUILD_ID = "BUILDBUCKET_BUILD_ID"
	VARIABLE_CODEREVIEW_SERVER    = "CODEREVIEW_SERVER"
	VARIABLE_ISSUE                = "ISSUE"
	VARIABLE_ISSUE_INT            = "ISSUE_INT"
	VARIABLE_ISSUE_SHORT          = "ISSUE_SHORT"
	VARIABLE_PATCH_REF            = "PATCH_REF"
	VARIABLE_PATCH_REPO           = "PATCH_REPO"
	VARIABLE_PATCH_STORAGE        = "PATCH_STORAGE"
	VARIABLE_PATCHSET             = "PATCHSET"
	VARIABLE_PATCHSET_INT         = "PATCHSET_INT"
	VARIABLE_REPO                 = "REPO"
	VARIABLE_REVISION             = "REVISION"
	VARIABLE_TASK_ID              = "TASK_ID"
	VARIABLE_TASK_NAME            = "TASK_NAME"
)

Variables

View Source 
var (
	// CIPD packages which may be used in tasks.
	CIPD_PKGS_GIT_LINUX_AMD64   = cipd.PkgsGit[cipd.PlatformLinuxAmd64]
	CIPD_PKGS_GIT_MAC_AMD64     = cipd.PkgsGit[cipd.PlatformMacAmd64]
	CIPD_PKGS_GIT_WINDOWS_AMD64 = cipd.PkgsGit[cipd.PlatformWindowsAmd64]
	CIPD_PKGS_GOLDCTL           = []*CipdPackage{cipd.MustGetPackage("skia/tools/goldctl/${platform}")}
	CIPD_PKGS_ISOLATE           = []*CipdPackage{
		cipd.MustGetPackage("infra/tools/luci/isolate/${platform}"),
	}
	CIPD_PKGS_PYTHON_LINUX_AMD64   = cipd.PkgsPython[cipd.PlatformLinuxAmd64]
	CIPD_PKGS_PYTHON_WINDOWS_AMD64 = cipd.PkgsPython[cipd.PlatformWindowsAmd64]

	CIPD_PKGS_KITCHEN_LINUX_AMD64 = append([]*CipdPackage{
		cipd.MustGetPackage("infra/tools/luci/kitchen/${platform}"),
		cipd.MustGetPackage("infra/tools/luci-auth/${platform}"),
	}, CIPD_PKGS_PYTHON_LINUX_AMD64...)

	// Variable placeholders; these are replaced with the actual value
	// at task triggering time.
	PLACEHOLDER_BUILDBUCKET_BUILD_ID = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_BUILDBUCKET_BUILD_ID)
	PLACEHOLDER_CODEREVIEW_SERVER    = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_CODEREVIEW_SERVER)
	PLACEHOLDER_ISSUE                = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_ISSUE)
	PLACEHOLDER_ISSUE_INT            = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_ISSUE_INT)
	PLACEHOLDER_ISSUE_SHORT          = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_ISSUE_SHORT)
	PLACEHOLDER_PATCH_REF            = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCH_REF)
	PLACEHOLDER_PATCH_REPO           = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCH_REPO)
	PLACEHOLDER_PATCH_STORAGE        = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCH_STORAGE)
	PLACEHOLDER_PATCHSET             = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCHSET)
	PLACEHOLDER_PATCHSET_INT         = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_PATCHSET_INT)
	PLACEHOLDER_REPO                 = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_REPO)
	PLACEHOLDER_REVISION             = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_REVISION)
	PLACEHOLDER_TASK_ID              = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_TASK_ID)
	PLACEHOLDER_TASK_NAME            = fmt.Sprintf(VARIABLE_SYNTAX, VARIABLE_TASK_NAME)
	PLACEHOLDER_ISOLATED_OUTDIR      = "${ISOLATED_OUTDIR}"

	PERIODIC_TRIGGERS = []string{TRIGGER_NIGHTLY, TRIGGER_WEEKLY}
)
View Source 
var (

	// EmptyCasSpec is a CasSpec with no contents.
	EmptyCasSpec = &CasSpec{
		Digest: rbe.EmptyDigest,
	}
)

Functions

func EncodeTasksCfg 

func EncodeTasksCfg(cfg *TasksCfg) ([]byte, error)

EncoderTasksCfg writes the TasksCfg to a byte slice.

func ErrorIsPermanent 

func ErrorIsPermanent(err error) bool

ErrorIsPermanent returns true if the given error cannot be recovered by retrying. In this case, we will never be able to process the TasksCfg, so we might as well cancel the jobs. TODO(borenet): This should probably be split into two different ErrorIsPermanent functions, in the syncer, and specs packages.

func GetCheckoutRoot 

func GetCheckoutRoot() (string, error)

GetCheckoutRoot returns the path of the root of the checkout.

func Python3LinuxAMD64CIPDPackages 

func Python3LinuxAMD64CIPDPackages() []*cipd.Package

func Python3WindowsAMD64CIPDPackages 

func Python3WindowsAMD64CIPDPackages() []*cipd.Package

func WriteTasksCfg 

func WriteTasksCfg(cfg *TasksCfg, repoDir string) error

WriteTasksCfg writes the task cfg to the given repo.

Types

type Cache 

type Cache struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

Cache is a struct representing a named cache which is used by a task.

type CasSpec 

type CasSpec struct {
	Root     string   `json:"root,omitempty"`
	Paths    []string `json:"paths,omitempty"`
	Excludes []string `json:"excludes,omitempty"`
	Digest   string   `json:"digest,omitempty"`
}

CasSpec describes a set of task inputs in content-addressed storage.

func (*CasSpec) Copy 

func (s *CasSpec) Copy() *CasSpec

Copy returns a deep copy of the CasSpec.

func (*CasSpec) Validate 

func (s *CasSpec) Validate() error

Validate returns an error if the CasSpec is invalid.

type CipdPackage 

type CipdPackage = cipd.Package

CipdPackage is a struct representing a CIPD package which needs to be installed on a bot for a particular task. TODO(borenet): Are there any downsides to using an alias rather than a new type here?

type CommitQueueJobConfig 

type CommitQueueJobConfig struct {
	// Run on the Commit Queue only if the change contains modifications to the
	// following location regexes.
	LocationRegexes []string `json:"location_regexes,omitempty"`
	// If this flag is true then the job is marked as being experimental. It will
	// be triggered on all CLs but their outcome will not affect the Commit Queue.
	// i.e. the experimental job could fail but if all other non-experimental job
	// have succeeded then the Commit Queue will succeed.
	Experimental bool `json:"experimental,omitempty"`
}

CommitQueueJobConfig describes how a job should run on the Commit Queue.

func (*CommitQueueJobConfig) Copy 

func (c *CommitQueueJobConfig) Copy() *CommitQueueJobConfig

Copy returns a deep copy of the CommitQueueJobConfig.

type JobSpec 

type JobSpec struct {
	// Priority indicates the relative priority of the job, with 0 < p <= 1,
	// where higher values result in scheduling the job's tasks sooner. If
	// unspecified or outside this range, DEFAULT_JOB_SPEC_PRIORITY is used.
	// Each task derives its priority from the set of jobs that depend upon
	// it. A task's priority is
	//   1 - (1-<job1 priority>)(1-<job2 priority>)...(1-<jobN priority>)
	// A task at HEAD with a priority of 1 and a blamelist of 1 commit has
	// approximately the same score as a task at HEAD with a priority of 0.57
	// and a blamelist of 2 commits.
	// A backfill task with a priority of 1 that bisects a blamelist of 2
	// commits has the same score as another backfill task at the same
	// commit with a priority of 0.4 that bisects a blamelist of 4 commits.
	Priority float64 `json:"priority,omitempty"`
	// The names of TaskSpecs that are direct dependencies of this JobSpec.
	TaskSpecs []string `json:"tasks"`
	// One of the TRIGGER_* constants; see documentation above.
	Trigger string `json:"trigger,omitempty"`
}

JobSpec is a struct which describes a set of TaskSpecs to run as part of a larger effort.

func (*JobSpec) Copy 

func (j *JobSpec) Copy() *JobSpec

Copy returns a copy of the JobSpec.

func (*JobSpec) GetTaskSpecDAG 

func (j *JobSpec) GetTaskSpecDAG(cfg *TasksCfg) (map[string][]string, error)

GetTaskSpecDAG returns a map describing all of the dependencies of the JobSpec. Its keys are TaskSpec names and values are TaskSpec names upon which the keys depend.

func (*JobSpec) Validate 

func (j *JobSpec) Validate() error

Validate returns an error if the JobSpec is not valid.

type TaskSpec 

type TaskSpec struct {
	// Caches are named Swarming caches which should be used for this task.
	Caches []*Cache `json:"caches,omitempty"`

	// CasSpec references a named input to the task from content-addressed
	// storage.
	CasSpec string `json:"casSpec,omitempty"`

	// CipdPackages are CIPD packages which should be installed for the task.
	CipdPackages []*CipdPackage `json:"cipd_packages,omitempty"`

	// Command is the command to run in the Swarming task.
	Command []string `json:"command,omitempty"`

	// Dependencies are names of other TaskSpecs for tasks which need to run
	// before this task.
	Dependencies []string `json:"dependencies,omitempty"`

	// Dimensions are Swarming bot dimensions which describe the type of bot
	// which may run this task.
	Dimensions []string `json:"dimensions"`

	// Environment is a set of environment variables needed by the task.
	Environment map[string]string `json:"environment,omitempty"`

	// EnvPrefixes are prefixes to add to environment variables for the task,
	// for example, adding directories to PATH. Keys are environment variable
	// names and values are multiple values to add for the variable.
	EnvPrefixes map[string][]string `json:"env_prefixes,omitempty"`

	// ExecutionTimeout is the maximum amount of time the task is allowed
	// to take.
	ExecutionTimeout time.Duration `json:"execution_timeout_ns,omitempty"`

	// Expiration is how long the task may remain in the pending state
	// before it is abandoned.
	Expiration time.Duration `json:"expiration_ns,omitempty"`

	// ExtraArgs are extra command-line arguments to pass to the task.
	// DEPRECATED: These are now ignored.
	ExtraArgs []string `json:"extra_args,omitempty"`

	// ExtraTags are extra tags to add to the Swarming task.
	ExtraTags map[string]string `json:"extra_tags,omitempty"`

	// Idempotent indicates that triggering this task with the same
	// parameters as previously triggered has no side effect and thus the
	// task may be de-duplicated.
	Idempotent bool `json:"idempotent,omitempty"`

	// IoTimeout is the maximum amount of time which the task may take to
	// communicate with the server.
	IoTimeout time.Duration `json:"io_timeout_ns,omitempty"`

	// MaxAttempts is the maximum number of attempts for this TaskSpec. If
	// zero, DEFAULT_TASK_SPEC_MAX_ATTEMPTS is used.
	MaxAttempts int `json:"max_attempts,omitempty"`

	// Outputs are files and/or directories to use as outputs for the task.
	// Paths are relative to the task workdir. No error occurs if any of
	// these is missing.
	Outputs []string `json:"outputs,omitempty"`

	// This field is ignored.
	Priority float64 `json:"priority,omitempty"`

	// ServiceAccount indicates the Swarming service account to use for the
	// task. If not specified, we will attempt to choose a suitable default.
	ServiceAccount string `json:"service_account,omitempty"`

	// TaskExecutor specifies what type of task executor should handle the task.
	TaskExecutor string `json:"task_executor,omitempty"`
}

TaskSpec is a struct which describes a Swarming task to run. Be sure to add any new fields to the Copy() method.

func (*TaskSpec) Copy 

func (t *TaskSpec) Copy() *TaskSpec

Copy returns a copy of the TaskSpec.

func (*TaskSpec) Validate 

func (t *TaskSpec) Validate(cfg *TasksCfg) error

Validate ensures that the TaskSpec is defined properly.

type TasksCfg 

type TasksCfg struct {
	// Jobs is a map whose keys are JobSpec names and values are JobSpecs
	// which describe sets of tasks to run.
	Jobs map[string]*JobSpec `json:"jobs"`

	// Tasks is a map whose keys are TaskSpec names and values are TaskSpecs
	// detailing the Swarming tasks which may be run.
	Tasks map[string]*TaskSpec `json:"tasks"`

	// CasSpecs is a map of named specifications for content-addressed inputs to
	// tasks.
	CasSpecs map[string]*CasSpec `json:"casSpecs,omitempty"`

	// CommitQueue is a map whose keys are JobSpec names and values are
	// CommitQueueJobConfig. All specified jobs will run on the Commit Queue.
	CommitQueue map[string]*CommitQueueJobConfig `json:"commit_queue,omitempty"`
}

TasksCfg is a struct which describes all Swarming tasks for a repo at a particular commit.

func ParseTasksCfg 

func ParseTasksCfg(contents string) (*TasksCfg, error)

ParseTasksCfg parses the given task cfg file contents and returns the config.

func ReadTasksCfg 

func ReadTasksCfg(repoDir string) (*TasksCfg, error)

ReadTasksCfg reads the task cfg file from the given dir and returns it.

func (*TasksCfg) Copy 

func (c *TasksCfg) Copy() *TasksCfg

Copy returns a deep copy of the TasksCfg.

func (*TasksCfg) Validate 

func (c *TasksCfg) Validate() error

Validate returns an error if the TasksCfg is not valid.

type TasksCfgBuilder 

type TasksCfgBuilder struct {
	// contains filtered or unexported fields
}

TasksCfgBuilder is a helper struct used for building a TasksCfg.

func MustNewTasksCfgBuilder 

func MustNewTasksCfgBuilder() *TasksCfgBuilder

MustNewTasksCfgBuilder returns a TasksCfgBuilder instance. Panics on error.

func NewTasksCfgBuilder 

func NewTasksCfgBuilder() (*TasksCfgBuilder, error)

NewTasksCfgBuilder returns a TasksCfgBuilder instance.

func (*TasksCfgBuilder) AddCQJob 

func (b *TasksCfgBuilder) AddCQJob(name string, c *CommitQueueJobConfig) error

AddCQJob adds a CommitQueueJobConfig to the TasksCfgBuilder.

func (*TasksCfgBuilder) AddCasSpec 

func (b *TasksCfgBuilder) AddCasSpec(name string, c *CasSpec) error

AddCasSpec adds a CasSpec to the TasksCfgBuilder.

func (*TasksCfgBuilder) AddJob 

func (b *TasksCfgBuilder) AddJob(name string, j *JobSpec) error

AddJob adds a JobSpec to the TasksCfgBuilder.

func (*TasksCfgBuilder) AddTask 

func (b *TasksCfgBuilder) AddTask(name string, t *TaskSpec) error

AddTask adds a TaskSpec to the TasksCfgBuilder. Returns an error if the config already contains a Task with the same name and a different implementation.

func (*TasksCfgBuilder) CheckoutRoot 

func (b *TasksCfgBuilder) CheckoutRoot() string

CheckoutRoot returns the path to the root of the client checkout.

func (*TasksCfgBuilder) Finish 

func (b *TasksCfgBuilder) Finish() error

Finish validates and writes out the TasksCfg, or, if the --test flag is provided, verifies that the contents have not changed.

func (*TasksCfgBuilder) GetCipdPackageFromAsset 

func (b *TasksCfgBuilder) GetCipdPackageFromAsset(assetName string) (*CipdPackage, error)

GetCipdPackageFromAsset reads the version information for the given asset and returns a CipdPackage instance.

func (*TasksCfgBuilder) MustAddCQJob 

func (b *TasksCfgBuilder) MustAddCQJob(name string, c *CommitQueueJobConfig)

MustAddCQJob adds a CommitQueueJobConfig to the TasksCfgBuilder and panics on failure.

func (*TasksCfgBuilder) MustAddCasSpec 

func (b *TasksCfgBuilder) MustAddCasSpec(name string, c *CasSpec)

MustAddCasSpec adds a CasSpec to the TasksCfgBuilder and panics on failure.

func (*TasksCfgBuilder) MustAddJob 

func (b *TasksCfgBuilder) MustAddJob(name string, j *JobSpec)

MustAddJob adds a JobSpec to the TasksCfgBuilder and panics on failure.

func (*TasksCfgBuilder) MustAddTask 

func (b *TasksCfgBuilder) MustAddTask(name string, t *TaskSpec)

MustAddTask adds a TaskSpec to the TasksCfgBuilder and panics on failure.

func (*TasksCfgBuilder) MustFinish 

func (b *TasksCfgBuilder) MustFinish()

MustFinish validates and writes out the TasksCfg, or, if the --test flag is provided, verifies that the contents have not changed. Panics on failure.

func (*TasksCfgBuilder) MustGetCipdPackageFromAsset 

func (b *TasksCfgBuilder) MustGetCipdPackageFromAsset(assetName string) *CipdPackage

MustGetCipdPackageFromAsset reads the version information for the given asset and returns a CipdPackage instance. Panics on failure.

func (*TasksCfgBuilder) SetAssetsDir 

func (b *TasksCfgBuilder) SetAssetsDir(assetsDir string)

SetAssetsDir sets the directory path used for assets.

Source Files

View all Source files

Directories

Path Synopsis
cmd
validate-tasks-json

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.