gen_tasks_logic

package
v0.0.0-...-23803d7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 25, 2025 License: BSD-3-Clause Imports: 21 Imported by: 1

Documentation

Overview

Copyright 2020 The Chromium Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Copyright 2020 The Chromium Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Copyright 2020 The Chromium Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Copyright 2020 The Chromium Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Index

Constants

View Source 
const (
	CAS_BAZEL         = "bazel"
	CAS_CANVASKIT     = "canvaskit"
	CAS_COMPILE       = "compile"
	CAS_EMPTY         = "empty" // TODO(borenet): It'd be nice if this wasn't necessary.
	CAS_LOTTIE_CI     = "lottie-ci"
	CAS_LOTTIE_WEB    = "lottie-web"
	CAS_PATHKIT       = "pathkit"
	CAS_PERF          = "perf"
	CAS_PUPPETEER     = "puppeteer"
	CAS_RUN_RECIPE    = "run-recipe"
	CAS_RECIPES       = "recipes"
	CAS_RECREATE_SKPS = "recreate-skps"
	CAS_SKOTTIE_WASM  = "skottie-wasm"
	CAS_TASK_DRIVERS  = "task-drivers"
	CAS_TEST          = "test"
	CAS_WASM_GM       = "wasm-gm"
	CAS_WHOLE_REPO    = "whole-repo"

	BUILD_TASK_DRIVERS_PREFIX  = "Housekeeper-PerCommit-BuildTaskDrivers"
	BUNDLE_RECIPES_NAME        = "Housekeeper-PerCommit-BundleRecipes"
	ISOLATE_GCLOUD_LINUX_NAME  = "Housekeeper-PerCommit-IsolateGCloudLinux"
	ISOLATE_SKIMAGE_NAME       = "Housekeeper-PerCommit-IsolateSkImage"
	ISOLATE_SKP_NAME           = "Housekeeper-PerCommit-IsolateSKP"
	ISOLATE_MSKP_NAME          = "Housekeeper-PerCommit-IsolateMSKP"
	ISOLATE_SVG_NAME           = "Housekeeper-PerCommit-IsolateSVG"
	ISOLATE_NDK_LINUX_NAME     = "Housekeeper-PerCommit-IsolateAndroidNDKLinux"
	ISOLATE_SDK_LINUX_NAME     = "Housekeeper-PerCommit-IsolateAndroidSDKLinux"
	ISOLATE_WIN_TOOLCHAIN_NAME = "Housekeeper-PerCommit-IsolateWinToolchain"

	DEBIAN_11_OS                   = "Debian-11.5"
	DEFAULT_OS_DEBIAN              = "Debian-10.10"
	DEFAULT_OS_LINUX_GCE           = "Debian-10.3"
	OLD_OS_LINUX_GCE               = "Debian-9.8"
	COMPILE_TASK_NAME_OS_LINUX     = "Debian10"
	COMPILE_TASK_NAME_OS_LINUX_OLD = "Debian9"
	DEFAULT_OS_MAC                 = "Mac-14.5"
	DEFAULT_OS_WIN_GCE             = "Windows-Server-17763"
	UBUNTU_20_04_OS                = "Ubuntu-20.04"
	UBUNTU_22_04_OS                = "Ubuntu-22.04"
	UBUNTU_24_04_OS                = "Ubuntu-24.04"

	// Small is a 2-core machine.
	// TODO(dogben): Would n1-standard-1 or n1-standard-2 be sufficient?
	MACHINE_TYPE_SMALL = "n1-highmem-2"
	// Medium is a 16-core machine
	MACHINE_TYPE_MEDIUM = "n1-standard-16"
	// Large is a 64-core machine. (We use "highcpu" because we don't need more than 57GB memory for
	// any of our tasks.)
	MACHINE_TYPE_LARGE = "n1-highcpu-64"

	// Swarming output dirs.
	OUTPUT_NONE          = "output_ignored" // This will result in outputs not being isolated.
	OUTPUT_BUILD         = "build"
	OUTPUT_BUILD_NOPATCH = "build_nopatch"
	OUTPUT_TEST          = "test"
	OUTPUT_PERF          = "perf"
	OUTPUT_BAZEL         = "bazel_output"

	// Name prefix for upload jobs.
	PREFIX_UPLOAD = "Upload"
)

Variables

View Source 
var (

	// Named caches used by tasks.
	CACHES_GIT = []*specs.Cache{
		{
			Name: "git",
			Path: "cache/git",
		},
		{
			Name: "git_cache",
			Path: "cache/git_cache",
		},
	}
	CACHES_GO = []*specs.Cache{
		{
			Name: "go_cache",
			Path: "cache/go_cache",
		},
		{
			Name: "gopath",
			Path: "cache/gopath",
		},
	}
	CACHES_WORKDIR = []*specs.Cache{
		{
			Name: "work",
			Path: "cache/work",
		},
	}
	CACHES_CCACHE = []*specs.Cache{
		{
			Name: "ccache",
			Path: "cache/ccache",
		},
	}
	// The "docker" cache is used as a persistent working directory for
	// tasks which use Docker. It is not to be confused with Docker's own
	// cache, which stores images. We do not currently use a named Swarming
	// cache for the latter.
	// TODO(borenet): We should ensure that any task which uses Docker does
	// not also use the normal "work" cache, to prevent issues like
	// https://bugs.chromium.org/p/skia/issues/detail?id=9749.
	CACHES_DOCKER = []*specs.Cache{
		{
			Name: "docker",
			Path: "cache/docker",
		},
	}

	// CAS_SPEC_LOTTIE_CI is a CasSpec which includes the files needed for
	// lottie-ci.  This is global so that it can be overridden by other
	// repositories which import this file.
	CAS_SPEC_LOTTIE_CI = &specs.CasSpec{
		Root: "..",
		Paths: []string{
			"skia/.vpython3",
			"skia/infra/bots/run_recipe.py",
			"skia/infra/lottiecap",
			"skia/tools/lottie-web-perf",
			"skia/tools/lottiecap",
		},
		Excludes: []string{rbe.ExcludeGitDir},
	}

	// CAS_SPEC_WHOLE_REPO is a CasSpec which includes the entire repo. This is
	// global so that it can be overridden by other repositories which import
	// this file.
	CAS_SPEC_WHOLE_REPO = &specs.CasSpec{
		Root:     "..",
		Paths:    []string{"skia"},
		Excludes: []string{rbe.ExcludeGitDir},
	}

	// TODO(borenet): This hacky and bad.
	CIPD_PKG_LUCI_AUTH = cipd.MustGetPackage("infra/tools/luci-auth/${platform}")

	CIPD_PKGS_GOLDCTL = cipd.MustGetPackage("skia/tools/goldctl/${platform}")

	CIPD_PKGS_XCODE = []*specs.CipdPackage{

		{
			Name: "infra/tools/mac_toolchain/${platform}",
			Path: "mac_toolchain",

			Version: "git_revision:e6f45bde6c5ee56924b1f905159b6a1a48ef25dd",
		},
	}

	// These properties are required by some tasks, eg. for running
	// bot_update, but they prevent de-duplication, so they should only be
	// used where necessary.
	EXTRA_PROPS = map[string]string{
		"buildbucket_build_id": specs.PLACEHOLDER_BUILDBUCKET_BUILD_ID,
		"patch_issue":          specs.PLACEHOLDER_ISSUE_INT,
		"patch_ref":            specs.PLACEHOLDER_PATCH_REF,
		"patch_repo":           specs.PLACEHOLDER_PATCH_REPO,
		"patch_set":            specs.PLACEHOLDER_PATCHSET_INT,
		"patch_storage":        specs.PLACEHOLDER_PATCH_STORAGE,
		"repository":           specs.PLACEHOLDER_REPO,
		"revision":             specs.PLACEHOLDER_REVISION,
		"task_id":              specs.PLACEHOLDER_TASK_ID,
	}

	// ISOLATE_ASSET_MAPPING maps the name of an asset to the configuration
	// for how the CIPD package should be installed for a given task.
	ISOLATE_ASSET_MAPPING = map[string]uploadAssetCASCfg{
		"gcloud_linux": {
			// contains filtered or unexported fields
		},
		"skimage": {
			// contains filtered or unexported fields
		},
		"skp": {
			// contains filtered or unexported fields
		},
		"svg": {
			// contains filtered or unexported fields
		},
		"mskp": {
			// contains filtered or unexported fields
		},
		"android_ndk_linux": {
			// contains filtered or unexported fields
		},
		"android_sdk_linux": {
			// contains filtered or unexported fields
		},
		"win_toolchain": {
			// contains filtered or unexported fields
		},
	}

	// Set dontReduceOpsTaskSplitting option on these models
	DONT_REDUCE_OPS_TASK_SPLITTING_MODELS = []string{
		"NUC5PPYH",
	}
)

Functions

func CheckoutRoot 

func CheckoutRoot() string

CheckoutRoot is a wrapper around specs.GetCheckoutRoot which prevents the caller from needing a dependency on the specs package.

func GenTasks 

func GenTasks(cfg *Config)

GenTasks regenerates the tasks.json file. Loads the job list from a jobs.json file which is the sibling of the calling gen_tasks.go file. If cfg is nil, it is similarly loaded from a cfg.json file which is the sibling of the calling gen_tasks.go file.

func In 

func In(s string, a []string) bool

In returns true if |s| is *in* |a| slice. TODO(borenet): This is copied from go.skia.org/infra/go/util to avoid the huge set of additional dependencies added by that package.

func LoadJson 

func LoadJson(filename string, dest interface{})

LoadJson loads JSON from the given file and unmarshals it into the given destination.

Types

type Config 

type Config struct {
	// Directory containing assets. Assumed to be relative to the directory
	// which contains the calling gen_tasks.go file. If not specified, uses
	// the infra/bots/assets from this repo.
	AssetsDir string `json:"assets_dir"`

	// Path to the builder name schema JSON file. Assumed to be relative to
	// the directory which contains the calling gen_tasks.go file. If not
	// specified, uses infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json
	// from this repo.
	BuilderNameSchemaFile string `json:"builder_name_schema"`

	// URL of the Skia Gold known hashes endpoint.
	GoldHashesURL string `json:"gold_hashes_url"`

	// GCS bucket used for GM results.
	GsBucketGm string `json:"gs_bucket_gm"`

	// GCS bucket used for Nanobench results.
	GsBucketNano string `json:"gs_bucket_nano"`

	// Optional function which returns a bot ID for internal devices.
	InternalHardwareLabel func(parts map[string]string) *int `json:"-"`

	// List of task names for which we'll never upload results.
	NoUpload []string `json:"no_upload"`

	// PathToSkia is the relative path from the root of the current checkout to
	// the root of the Skia checkout.
	PathToSkia string `json:"path_to_skia"`

	// Swarming pool used for triggering tasks.
	Pool string `json:"pool"`

	// LUCI project associated with this repo.
	Project string `json:"project"`

	// Service accounts.
	ServiceAccountCanary       string `json:"service_account_canary"`
	ServiceAccountCompile      string `json:"service_account_compile"`
	ServiceAccountHousekeeper  string `json:"service_account_housekeeper"`
	ServiceAccountRecreateSKPs string `json:"service_account_recreate_skps"`
	ServiceAccountUploadBinary string `json:"service_account_upload_binary"`
	ServiceAccountUploadGM     string `json:"service_account_upload_gm"`
	ServiceAccountUploadNano   string `json:"service_account_upload_nano"`

	// Optional override function which derives Swarming bot dimensions
	// from parts of task names.
	SwarmDimensions func(parts map[string]string) []string `json:"-"`
}

Config contains general configuration information.

func LoadConfig 

func LoadConfig() *Config

LoadConfig loads the Config from a cfg.json file which is the sibling of the calling gen_tasks.go file.

type JobInfo 

type JobInfo struct {
	// The name of the job.
	Name string `json:"name"`

	// The optional CQ config of this job. If the CQ config is missing then the
	// job will not be added to the CQ of this branch.
	CQConfig *specs.CommitQueueJobConfig `json:"cq_config,omitempty"`
}

JobInfo is the type of each entry in the jobs.json file.

type JobNameSchema 

type JobNameSchema struct {
	Schema map[string]*schema `json:"builder_name_schema"`
	Sep    string             `json:"builder_name_sep"`
}

JobNameSchema is a struct used for (de)constructing Job names in a predictable format.

func NewJobNameSchema 

func NewJobNameSchema(jsonFile string) (*JobNameSchema, error)

NewJobNameSchema returns a JobNameSchema instance based on the given JSON file.

func (*JobNameSchema) MakeJobName 

func (s *JobNameSchema) MakeJobName(parts map[string]string) (string, error)

MakeJobName assembles the given parts of a Job name, according to the schema.

func (*JobNameSchema) ParseJobName 

func (s *JobNameSchema) ParseJobName(n string) (map[string]string, error)

ParseJobName splits the given Job name into its component parts, according to the schema.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.