gce

Documentation

Index

• Constants
• Variables
• type Disk
• type GCloud
  • func NewGCloud(project, zone string, tokenSource oauth2.TokenSource) (*GCloud, error)
  • func NewLocalGCloud(project, zone string) (*GCloud, error)
  • func (g *GCloud) CaptureImage(vm *Instance, family, description string) error
  • func (g *GCloud) CheckOperation(op *compute.Operation, err error) error
  • func (g *GCloud) CheckSsh() error
  • func (g *GCloud) CreateAndSetup(ctx context.Context, vm *Instance, ignoreExists bool) error
  • func (g *GCloud) CreateDisk(disk *Disk, ignoreExists bool) error
  • func (g *GCloud) Delete(vm *Instance, ignoreNotExists, deleteDataDisk bool) error
  • func (g *GCloud) DeleteDisk(name string, ignoreNotExists bool) error
  • func (g *GCloud) DeleteInstance(name string, ignoreNotExists bool) error
  • func (g *GCloud) DownloadFile(ctx context.Context, vm *Instance, f *GSDownload) error
  • func (g *GCloud) GetFileFromMetadata(ctx context.Context, vm *Instance, url, dst string) error
  • func (g *GCloud) GetImages() ([]*compute.Image, error)
  • func (g *GCloud) GetIpAddress(vm *Instance) (string, error)
  • func (g *GCloud) InstallWget(ctx context.Context, vm *Instance) error
  • func (g *GCloud) IsInstanceReady(ctx context.Context, vm *Instance) (bool, error)
  • func (g *GCloud) IsInstanceRunning(name string) (bool, error)
  • func (g *GCloud) ListDisks() ([]*Disk, error)
  • func (g *GCloud) Reboot(ctx context.Context, vm *Instance) error
  • func (g *GCloud) SafeFormatAndMount(ctx context.Context, vm *Instance) error
  • func (g *GCloud) Scp(ctx context.Context, vm *Instance, src, dst string) error
  • func (g *GCloud) Service() *compute.Service
  • func (g *GCloud) SetMetadata(vm *Instance, md map[string]string) error
  • func (g *GCloud) Ssh(ctx context.Context, vm *Instance, cmd ...string) (string, error)
  • func (g *GCloud) Start(ctx context.Context, vm *Instance) error
  • func (g *GCloud) StartWithoutReadyCheck(vm *Instance) error
  • func (g *GCloud) Stop(vm *Instance) error
  • func (g *GCloud) WaitForInstanceReady(ctx context.Context, vm *Instance, timeout time.Duration) error
  • func (g *GCloud) WaitForLogMessage(vm *Instance, logText string, timeout time.Duration) error
• type GSDownload
• type Instance

Constants

const (
	CPU_PLATFORM_AMD     = "AMD Rome"
	CPU_PLATFORM_SKYLAKE = "Intel Skylake"

	// Labels can only contain lowercase letters, numbers, underscores, and dashes.
	DATE_FORMAT     = "2006-01-02"
	DATETIME_FORMAT = "2006-01-02_15-04-05"

	DISK_MOUNT_PATH_DEFAULT = "/mnt/pd0"

	DISK_SNAPSHOT_SYSTEMD_PUSHABLE_BASE = "skia-systemd-pushable-base"

	DISK_TYPE_LOCAL_SSD           = "local-ssd"
	DISK_TYPE_PERSISTENT_STANDARD = "pd-standard"
	DISK_TYPE_PERSISTENT_SSD      = "pd-ssd"

	IMAGE_STATUS_READY = "READY"

	MACHINE_TYPE_F1_MICRO    = "f1-micro"
	MACHINE_TYPE_HIGHCPU_64  = "n1-highcpu-64"
	MACHINE_TYPE_HIGHMEM_16  = "n1-highmem-16"
	MACHINE_TYPE_HIGHMEM_2   = "n1-highmem-2"
	MACHINE_TYPE_HIGHMEM_4   = "n1-highmem-4"
	MACHINE_TYPE_HIGHMEM_32  = "n1-highmem-32"
	MACHINE_TYPE_HIGHMEM_64  = "n1-highmem-64"
	MACHINE_TYPE_STANDARD_1  = "n1-standard-1"
	MACHINE_TYPE_STANDARD_16 = "n1-standard-16"
	MACHINE_TYPE_STANDARD_2  = "n1-standard-2"
	MACHINE_TYPE_STANDARD_32 = "n1-standard-32"
	MACHINE_TYPE_STANDARD_4  = "n1-standard-4"
	MACHINE_TYPE_STANDARD_8  = "n1-standard-8"
	// https://cloud.google.com/compute/docs/machine-types#n2d_machine_types
	MACHINE_TYPE_N2D_STANDARD_16 = "n2d-standard-16"

	MAINTENANCE_POLICY_MIGRATE   = "MIGRATE"
	MAINTENANCE_POLICY_TERMINATE = "TERMINATE"

	NETWORK_DEFAULT = "global/networks/default"

	OPERATION_STATUS_DONE = "DONE"

	OS_LINUX   = "Linux"
	OS_WINDOWS = "Windows"

	PROJECT_ID_SERVER            = common.PROJECT_ID
	PROJECT_ID_SWARMING          = "skia-swarming-bots"
	PROJECT_ID_CT_SWARMING       = "ct-swarming-bots"
	PROJECT_ID_INTERNAL_SWARMING = "skia-swarming-bots-internal"

	SERVICE_ACCOUNT_DEFAULT         = "31977622648@project.gserviceaccount.com"
	SERVICE_ACCOUNT_COMPUTE         = "31977622648-compute@developer.gserviceaccount.com"
	SERVICE_ACCOUNT_CHROME_SWARMING = "chrome-swarming-bots@skia-swarming-bots-internal.iam.gserviceaccount.com"
	SERVICE_ACCOUNT_CHROMIUM_SWARM  = "chromium-swarm-bots@skia-swarming-bots.iam.gserviceaccount.com"
	SERVICE_ACCOUNT_CT_SWARMING     = "ct-swarming-bots@ct-swarming-bots.iam.gserviceaccount.com"

	SETUP_SCRIPT_KEY_LINUX  = "setup-script"
	SETUP_SCRIPT_KEY_WIN    = "sysprep-specialize-script-ps1"
	SETUP_SCRIPT_PATH_LINUX = "/tmp/setup-script.sh"

	USER_CHROME_BOT = "chrome-bot"
	USER_DEFAULT    = "default"

	ZONE_CENTRAL1_A = "us-central1-a"
	ZONE_CENTRAL1_B = "us-central1-b"
	ZONE_CENTRAL1_C = "us-central1-c"
	ZONE_EAST1_D    = "us-east1-d"

	ZONE_CT      = ZONE_CENTRAL1_B
	ZONE_DEFAULT = ZONE_CENTRAL1_C
	ZONE_SKYLAKE = ZONE_CENTRAL1_B
	// On 5/20/20 "AMD Rome" was only available on us-central1-a and us-central1-f in us-central (https://cloud.google.com/compute/docs/regions-zones#available)
	ZONE_AMD = ZONE_CENTRAL1_A
)

Variables

var (
	AUTH_SCOPES = []string{compute.CloudPlatformScope, compute.ComputeScope, compute.DevstorageFullControlScope}
	VALID_OS    = []string{OS_LINUX, OS_WINDOWS}
	VALID_ZONES = []string{ZONE_CENTRAL1_B, ZONE_CENTRAL1_C, ZONE_EAST1_D}
)

Types

type Disk

type Disk struct {
	// The name of the disk.
	Name string

	// Size of the disk, in gigabytes.
	SizeGb int64

	// Optional, image to flash to the disk. Use only one of SourceImage
	// and SourceSnapshot.
	SourceImage string

	// Optional, snapshot to flash to the disk. Use only one of SourceImage
	// and SourceSnapshot.
	SourceSnapshot string

	// Type of disk, eg. "pd-standard" or "pd-ssd".
	Type string

	// Output only, which instances are using this disk, if any.
	InUseBy []string

	// Optional mountpoint. Default: /mnt/pd0 (see format_and_mount.sh)
	MountPath string
}

Disk is a struct describing a disk resource in GCE.

type GCloud

type GCloud struct {
	// contains filtered or unexported fields
}

GCloud is a struct used for creating disks and instances in GCE.

func NewGCloud

func NewGCloud(project, zone string, tokenSource oauth2.TokenSource) (*GCloud, error)

NewGCloud returns a GCloud instance with the specified http client.

func NewLocalGCloud

func NewLocalGCloud(project, zone string) (*GCloud, error)

NewLocalGCloud returns a GCloud instance with a default http client for local use.

func (*GCloud) CaptureImage

func (g *GCloud) CaptureImage(vm *Instance, family, description string) error

CaptureImage captures an image from the instance's boot disk. The instance has to be deleted in order to capture the image, and we delete the boot disk after capture for cleanliness.

func (*GCloud) CheckOperation

func (g *GCloud) CheckOperation(op *compute.Operation, err error) error

CheckOperation waits for the given operation to complete and returns any error. Convenience wrapper around waitForOperation.

func (*GCloud) CheckSsh

func (g *GCloud) CheckSsh() error

CheckSsh returns an error if we are unable to use SSH.

func (*GCloud) CreateAndSetup

func (g *GCloud) CreateAndSetup(ctx context.Context, vm *Instance, ignoreExists bool) error

CreateAndSetup creates an instance and all its disks and performs any additional setup steps.

func (*GCloud) CreateDisk

func (g *GCloud) CreateDisk(disk *Disk, ignoreExists bool) error

CreateDisk creates the given disk.

func (*GCloud) Delete

func (g *GCloud) Delete(vm *Instance, ignoreNotExists, deleteDataDisk bool) error

Delete removes the instance and (maybe) its disks.

func (*GCloud) DeleteDisk

func (g *GCloud) DeleteDisk(name string, ignoreNotExists bool) error

DeleteDisk deletes the given disk.

func (*GCloud) DeleteInstance

func (g *GCloud) DeleteInstance(name string, ignoreNotExists bool) error

DeleteInstance deletes the given GCE VM instance.

func (*GCloud) DownloadFile

func (g *GCloud) DownloadFile(ctx context.Context, vm *Instance, f *GSDownload) error

DownloadFile downloads the given file from Google Cloud Storage to the instance.

func (*GCloud) GetFileFromMetadata

func (g *GCloud) GetFileFromMetadata(ctx context.Context, vm *Instance, url, dst string) error

GetFileFromMetadata downloads the given metadata entry to a file.

func (*GCloud) GetImages

func (g *GCloud) GetImages() ([]*compute.Image, error)

GetImages returns all of the images from the project.

func (*GCloud) GetIpAddress

func (g *GCloud) GetIpAddress(vm *Instance) (string, error)

GetIpAddress obtains the IP address for the Instance.

func (*GCloud) InstallWget

func (g *GCloud) InstallWget(ctx context.Context, vm *Instance) error

InstallWget installs wget CLI to be used by the above.

func (*GCloud) IsInstanceReady

func (g *GCloud) IsInstanceReady(ctx context.Context, vm *Instance) (bool, error)

IsInstanceReady returns true iff the instance is ready.

func (*GCloud) IsInstanceRunning

func (g *GCloud) IsInstanceRunning(name string) (bool, error)

IsInstanceRunning returns whether the instance is in running state.

func (*GCloud) ListDisks

func (g *GCloud) ListDisks() ([]*Disk, error)

ListDisks returns a list of Disks in the project.

func (*GCloud) Reboot

func (g *GCloud) Reboot(ctx context.Context, vm *Instance) error

Reboot stops and starts the instance. Returns when the instance is ready to use.

func (*GCloud) SafeFormatAndMount

func (g *GCloud) SafeFormatAndMount(ctx context.Context, vm *Instance) error

SafeFormatAndMount copies the safe_format_and_mount script to the instance and runs it for all data disks.

func (*GCloud) Scp

func (g *GCloud) Scp(ctx context.Context, vm *Instance, src, dst string) error

Scp copies files to the instance. The src argument is expected to be absolute.

func (*GCloud) Service

func (g *GCloud) Service() *compute.Service

Service returns the underlying compute.Service instance.

func (*GCloud) SetMetadata

func (g *GCloud) SetMetadata(vm *Instance, md map[string]string) error

SetMetadata sets the given metadata on the instance.

func (*GCloud) Ssh

func (g *GCloud) Ssh(ctx context.Context, vm *Instance, cmd ...string) (string, error)

Ssh logs into the instance and runs the given command. Returns any output and an error if applicable.

func (*GCloud) Start

func (g *GCloud) Start(ctx context.Context, vm *Instance) error

Stop stops the instance and returns when the instance is ready (ssh-able).

func (*GCloud) StartWithoutReadyCheck

func (g *GCloud) StartWithoutReadyCheck(vm *Instance) error

StartWithoutReadyCheck starts the instance and returns when the instance is in RUNNING state. Note: This method does not wait for the instance to be ready (ssh-able).

func (*GCloud) Stop

func (g *GCloud) Stop(vm *Instance) error

Stop stops the instance and returns when the operation completes.

func (*GCloud) WaitForInstanceReady

func (g *GCloud) WaitForInstanceReady(ctx context.Context, vm *Instance, timeout time.Duration) error

WaitForInstanceReady waits until the instance is ready to use.

func (*GCloud) WaitForLogMessage

func (g *GCloud) WaitForLogMessage(vm *Instance, logText string, timeout time.Duration) error

WaitForLogMessage waits until the given string appears in the instance's log. Currently only implemented for Windows.

type GSDownload

type GSDownload struct {
	// Full GS path of the file to download.
	Source string
	// Absolute or relative (to the user's home dir) destination path of the
	// file to download.
	Dest string
	// Mode, as accepted by the chmod command, for the file. If not
	// specified, the default file mode is used.
	Mode string
}

A file to be downloaded from GS.

type Instance

type Instance struct {
	// Information about the boot disk. Required.
	BootDisk *Disk

	// Email addresses of interested parties. Optional.
	Contacts []string

	// Information about an extra data disk. Optional.
	DataDisks []*Disk

	// External IP address for the instance. Required.
	ExternalIpAddress string

	// Files to download from Google Storage.
	GSDownloads []*GSDownload

	// GCE machine type specification, eg. "n1-standard-16".
	MachineType string

	// Maintenance policy. Default is MAINTENANCE_POLICY_MIGRATE, which is
	// not supported for preemtible instances.
	MaintenancePolicy string

	// Instance-level metadata keys and values.
	Metadata map[string]string

	// Files to create based on metadata. Map keys are destination paths on
	// the GCE instance and values are the source URLs (see
	// metadata.METADATA_URL). Paths May be absolute or relative (to the
	// default user's home dir, eg. /home/default).
	MetadataDownloads map[string]string

	// Minimum CPU platform, eg. CPU_PLATFORM_SKYLAKE.  Default is
	// determined by GCE.
	MinCpuPlatform string

	// Name of the instance.
	Name string

	// Operating system of the instance.
	Os string

	// Auth scopes for the instance.
	Scopes []string

	// Contents of a setup script for the instance, optional. The setup script
	// runs once after the instance is created. For Windows, this is assumed to
	// be a PowerShell script and runs during sysprep. For Linux, the script
	// needs to be executable via the shell (ie. use a shebang for Python
	// scripts).
	SetupScript string

	// The service account to use for this instance. Required.
	ServiceAccount string

	// Contents of a startup script for the instance, optional. The startup
	// script runs as root every time the instance starts up. For Windows, this
	// is assumed to be a PowerShell script. For Linux, the script needs to be
	// executable via the shell (ie. use a shebang for Python scripts).
	StartupScript string

	// Tags for the instance.
	Tags []string

	// Default user name for the instance.
	User string
}

Instance is a struct representing a GCE VM instance.