cloud

package
v0.0.0-...-418c65a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 26, 2019 License: Apache-2.0 Imports: 59 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	//Catch-all for unrecognized status codes
	StatusUnknown = CloudStatus(iota)

	//StatusPending indicates that it is not yet clear if the instance
	//has been successfully started or not (e.g., pending spot request)
	StatusPending

	//StatusInitializing means the instance request has been successfully
	//fulfilled, but it's not yet done booting up
	StatusInitializing

	//StatusFailed indicates that an attempt to start the instance has failed;
	//Could be due to billing, lack of capacity, etc.
	StatusFailed

	//StatusRunning means the machine is done booting, and active
	StatusRunning

	StatusStopped
	StatusTerminated
)
View Source 
const (
	SpotStatusOpen     = "open"
	SpotStatusActive   = "active"
	SpotStatusClosed   = "closed"
	SpotStatusCanceled = "cancelled"
	SpotStatusFailed   = "failed"

	EC2ErrorSpotRequestNotFound = "InvalidSpotInstanceRequestID.NotFound"
)
View Source 
const (
	EC2StatusPending      = "pending"
	EC2StatusRunning      = "running"
	EC2StatusShuttingdown = "shutting-down"
	EC2StatusTerminated   = "terminated"
	EC2StatusStopped      = "stopped"
	EC2ErrorNotFound      = "InvalidInstanceID.NotFound"
)

Valid values for EC2 instance states: pending | running | shutting-down | terminated | stopping | stopped see http://goo.gl/3OrCGn

View Source 
const (
	// MinUptime is the minimum time to run a host before shutting it down.
	MinUptime = 30 * time.Minute
	// BufferTime is the time to leave an idle host running after it has completed
	// a task before shutting it down, given that it has already run for MinUptime.
	BufferTime = 10 * time.Minute
)
View Source 
const (
	// OSStatusActive means the instance is currently running.
	OSStatusActive = "ACTIVE"
	// OSStatusInProgress means the instance is currently running and processing a request.
	OSStatusInProgress = "IN_PROGRESS"
	// OSStatusShutOff means the instance has been temporarily stopped.
	OSStatusShutOff = "SHUTOFF"
	// OSStatusBuilding means the instance is starting up.
	OSStatusBuilding = "BUILD"
)
View Source 
const (
	MaxSpawnHostsPerUser                = 3
	DefaultSpawnHostExpiration          = 24 * time.Hour
	MaxSpawnHostExpirationDurationHours = 24 * time.Hour * 14
)
View Source 
const MockIPV6 = "abcd:1234:459c:2d00:cfe4:843b:1d60:8e47"
View Source 
const (
	// ProviderName is used to distinguish between different cloud providers.
	ProviderName = "gce"
)

Variables

View Source 
var (
	// bson fields for the EC2ProviderSettings struct
	AMIKey            = bsonutil.MustHaveTag(EC2ProviderSettings{}, "AMI")
	InstanceTypeKey   = bsonutil.MustHaveTag(EC2ProviderSettings{}, "InstanceType")
	SecurityGroupsKey = bsonutil.MustHaveTag(EC2ProviderSettings{}, "SecurityGroupIDs")
	KeyNameKey        = bsonutil.MustHaveTag(EC2ProviderSettings{}, "KeyName")
	MountPointsKey    = bsonutil.MustHaveTag(EC2ProviderSettings{}, "MountPoints")
)
View Source 
var (
	// bson fields for the MountPoint struct
	VirtualNameKey = bsonutil.MustHaveTag(MountPoint{}, "VirtualName")
	DeviceNameKey  = bsonutil.MustHaveTag(MountPoint{}, "DeviceName")
	SizeKey        = bsonutil.MustHaveTag(MountPoint{}, "Size")
	VolumeTypeKey  = bsonutil.MustHaveTag(MountPoint{}, "VolumeType")
)
View Source 
var (
	// bson fields for the StaticSettings struct
	HostsKey = bsonutil.MustHaveTag(StaticSettings{}, "Hosts")

	// bson fields for the Host struct
	NameKey = bsonutil.MustHaveTag(StaticHost{}, "Name")
)
View Source 
var (
	// bson fields for the EC2SpotSettings struct
	BidPriceKey = bsonutil.MustHaveTag(EC2ProviderSettings{}, "BidPrice")
)

Functions

func CreateSpawnHost 

func CreateSpawnHost(so SpawnOptions) (*host.Host, error)

CreateHost spawns a host with the given options.

func GetEC2Key 

func GetEC2Key(region string, s *evergreen.Settings) (string, string, error)

Get EC2 key and secret from the AWS configuration for the given region

func GetMockClient 

func GetMockClient() *dockerClientMock

func GetRegion 

func GetRegion(d distro.Distro) string

GetRegion gets the region name from the provider settings object for a given provider name.

func GroupHostsByManager 

func GroupHostsByManager(hosts []host.Host) map[ManagerOpts][]host.Host

GroupHostsByManager returns a map where the key represents the manager options (provider name and region) and the value represents the hosts from the provided slice that are reachable through that manager.

func IsEc2Provider 

func IsEc2Provider(provider string) bool

func MakeExtendedSpawnHostExpiration 

func MakeExtendedSpawnHostExpiration(host *host.Host, extendBy time.Duration) (time.Time, error)

func SetHostRDPPassword 

func SetHostRDPPassword(ctx context.Context, env evergreen.Environment, host *host.Host, password string) error

func TerminateSpawnHost 

func TerminateSpawnHost(ctx context.Context, host *host.Host, settings *evergreen.Settings, user string) error

func ValidateRDPPassword 

func ValidateRDPPassword(password string) bool

XXX: if modifying any of the password validation logic, you changes must also be ported into public/static/js/directives/directives.spawn.js

Types

type AWSClient 

type AWSClient interface {
	// Create a new aws-sdk-client or mock if one does not exist, otherwise no-op.
	Create(*credentials.Credentials, string) error

	// Close an aws-sdk-client or mock.
	Close()

	// RunInstances is a wrapper for ec2.RunInstances.
	RunInstances(context.Context, *ec2.RunInstancesInput) (*ec2.Reservation, error)

	// DescribeInstances is a wrapper for ec2.DescribeInstances.
	DescribeInstances(context.Context, *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error)

	// CreateTags is a wrapper for ec2.CreateTags.
	CreateTags(context.Context, *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error)

	// DeleteTags is a wrapper for ec2.DeleteTags.
	DeleteTags(context.Context, *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error)

	// TerminateInstances is a wrapper for ec2.TerminateInstances.
	TerminateInstances(context.Context, *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error)

	// RequestSpotInstances is a wrapper for ec2.RequestSpotInstances.
	RequestSpotInstances(context.Context, *ec2.RequestSpotInstancesInput) (*ec2.RequestSpotInstancesOutput, error)

	// DescribeSpotInstanceRequests is a wrapper for ec2.DescribeSpotInstanceRequests.
	DescribeSpotInstanceRequests(ctx context.Context, input *ec2.DescribeSpotInstanceRequestsInput) (*ec2.DescribeSpotInstanceRequestsOutput, error)

	// DescribeSpotRequestsAndSave is a wrapper for DescribeSpotInstanceRequests that also saves instance IDs to the db.
	DescribeSpotRequestsAndSave(context.Context, []*host.Host) (*ec2.DescribeSpotInstanceRequestsOutput, error)

	// GetSpotInstanceId returns the instance ID if already saved, otherwise looks it up.
	GetSpotInstanceId(context.Context, *host.Host) (string, error)

	// CancelSpotInstanceRequests is a wrapper for ec2.CancelSpotInstanceRequests.
	CancelSpotInstanceRequests(context.Context, *ec2.CancelSpotInstanceRequestsInput) (*ec2.CancelSpotInstanceRequestsOutput, error)

	// DescribeVolumes is a wrapper for ec2.DescribeVolumes.
	DescribeVolumes(context.Context, *ec2.DescribeVolumesInput) (*ec2.DescribeVolumesOutput, error)

	// DescribeSpotPriceHistory is a wrapper for ec2.DescribeSpotPriceHistory.
	DescribeSpotPriceHistory(context.Context, *ec2.DescribeSpotPriceHistoryInput) (*ec2.DescribeSpotPriceHistoryOutput, error)

	// DescribeSubnets is a wrapper for ec2.DescribeSubnets.
	DescribeSubnets(context.Context, *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error)

	// DescribeVpcs is a wrapper for ec2.DescribeVpcs.
	DescribeVpcs(context.Context, *ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error)

	// GetInstanceInfo returns info about an ec2 instance.
	GetInstanceInfo(context.Context, string) (*ec2.Instance, error)

	// CreateKeyPair is a wrapper for ec2.CreateKeyPairWithContext.
	CreateKeyPair(context.Context, *ec2.CreateKeyPairInput) (*ec2.CreateKeyPairOutput, error)

	// DeleteKeyPair is a wrapper for ec2.DeleteKeyPairWithContext.
	DeleteKeyPair(context.Context, *ec2.DeleteKeyPairInput) (*ec2.DeleteKeyPairOutput, error)

	// GetProducts is a wrapper for pricing.GetProducts.
	GetProducts(context.Context, *pricing.GetProductsInput) (*pricing.GetProductsOutput, error)

	// CreateLaunchTemplate is a wrapper for ec2.CreateLaunchTemplateWithContext
	CreateLaunchTemplate(context.Context, *ec2.CreateLaunchTemplateInput) (*ec2.CreateLaunchTemplateOutput, error)

	// DeleteLaunchTemplate is a wrapper for ec2.DeleteLaunchTemplateWithContext
	DeleteLaunchTemplate(context.Context, *ec2.DeleteLaunchTemplateInput) (*ec2.DeleteLaunchTemplateOutput, error)

	// CreateFleet is a wrapper for ec2.CreateFleetWithContext
	CreateFleet(context.Context, *ec2.CreateFleetInput) (*ec2.CreateFleetOutput, error)

	GetKey(context.Context, *host.Host) (string, error)

	SetTags(context.Context, []string, *host.Host) error

	GetVolumeIDs(context.Context, *host.Host) ([]string, error)

	GetPublicDNSName(ctx context.Context, h *host.Host) (string, error)
}

AWSClient is a wrapper for aws-sdk-go so we can use a mock in testing.

type BatchManager 

type BatchManager interface {
	// GetInstanceStatuses gets the status of a slice of instances.
	GetInstanceStatuses(context.Context, []host.Host) ([]CloudStatus, error)
}

BatchManager is an interface for cloud providers that support batch operations.

type CloudHost 

type CloudHost struct {
	Host     *host.Host
	KeyPath  string
	CloudMgr Manager
}

CloudHost is a provider-agnostic host object that delegates methods like status checks, ssh options, DNS name checks, termination, etc. to the underlying provider's implementation.

func GetCloudHost 

func GetCloudHost(ctx context.Context, host *host.Host, settings *evergreen.Settings) (*CloudHost, error)

GetCloudHost returns an instance of CloudHost wrapping the given model.Host, giving access to the provider-specific methods to manipulate on the host.

func (*CloudHost) GetDNSName 

func (cloudHost *CloudHost) GetDNSName(ctx context.Context) (string, error)

func (*CloudHost) GetInstanceStatus 

func (cloudHost *CloudHost) GetInstanceStatus(ctx context.Context) (CloudStatus, error)

func (*CloudHost) GetSSHOptions 

func (cloudHost *CloudHost) GetSSHOptions() ([]string, error)

func (*CloudHost) IsUp 

func (cloudHost *CloudHost) IsUp(ctx context.Context) (bool, error)

func (*CloudHost) TerminateInstance 

func (cloudHost *CloudHost) TerminateInstance(ctx context.Context, user string) error

type CloudStatus 

type CloudStatus int

func (CloudStatus) String 

func (stat CloudStatus) String() string

type ContainerManager 

type ContainerManager interface {
	Manager

	// GetContainers returns the IDs of all running containers on a specified host
	GetContainers(context.Context, *host.Host) ([]string, error)
	// RemoveOldestImage removes the earliest created image on a specified host
	RemoveOldestImage(ctx context.Context, h *host.Host) error
	// CalculateImageSpaceUsage returns the total space taken up by docker images on a specified host
	CalculateImageSpaceUsage(ctx context.Context, h *host.Host) (int64, error)
	// GetContainerImage downloads a container image onto parent specified by URL, and builds evergreen agent unless otherwise specified
	GetContainerImage(ctx context.Context, parent *host.Host, options host.DockerOptions) error
}

func ConvertContainerManager 

func ConvertContainerManager(m Manager) (ContainerManager, error)

ConvertContainerManager converts a regular manager into a container manager, errors if type conversion not possible.

type ContainerStatus 

type ContainerStatus struct {
	IsRunning  bool
	HasStarted bool
}

type CostCalculator 

type CostCalculator interface {
	CostForDuration(context.Context, *host.Host, time.Time, time.Time, *evergreen.Settings) (float64, error)
}

CostCalculator is an interface for cloud providers that can estimate what a span of time on a given host costs.

type DockerClient 

type DockerClient interface {
	Init(string) error
	EnsureImageDownloaded(context.Context, *host.Host, host.DockerOptions) (string, error)
	BuildImageWithAgent(context.Context, *host.Host, string) (string, error)
	CreateContainer(context.Context, *host.Host, *host.Host) error
	GetContainer(context.Context, *host.Host, string) (*types.ContainerJSON, error)
	GetDockerLogs(context.Context, string, *host.Host, types.ContainerLogsOptions) (io.Reader, error)
	GetDockerStatus(context.Context, string, *host.Host) (*ContainerStatus, error)
	ListContainers(context.Context, *host.Host) ([]types.Container, error)
	RemoveImage(context.Context, *host.Host, string) error
	RemoveContainer(context.Context, *host.Host, string) error
	StartContainer(context.Context, *host.Host, string) error
	ListImages(context.Context, *host.Host) ([]types.ImageSummary, error)
}

The DockerClient interface wraps the Docker dockerClient interaction.

func GetDockerClient 

func GetDockerClient(s *evergreen.Settings) DockerClient

type EC2FleetManagerOptions 

type EC2FleetManagerOptions struct {
	// contains filtered or unexported fields
}

type EC2ManagerOptions 

type EC2ManagerOptions struct {
	// contains filtered or unexported fields
}

EC2ManagerOptions are used to construct a new ec2Manager.

type EC2ProviderSettings 

type EC2ProviderSettings struct {
	// AMI is the AMI ID.
	AMI string `mapstructure:"ami" json:"ami,omitempty" bson:"ami,omitempty"`

	// If set, overrides key from credentials
	AWSKeyID string `mapstructure:"aws_access_key_id" bson:"aws_access_key_id,omitempty"`

	// If set, overrides secret from credentials
	AWSSecret string `mapstructure:"aws_secret_access_key" bson:"aws_secret_access_key,omitempty"`

	// InstanceType is the EC2 instance type.
	InstanceType string `mapstructure:"instance_type" json:"instance_type,omitempty" bson:"instance_type,omitempty"`

	// IPv6 is set to true if the instance should have only an IPv6 address.
	IPv6 bool `mapstructure:"ipv6" json:"ipv6,omitempty" bson:"ipv6,omitempty"`

	// KeyName is the AWS SSH key name.
	KeyName string `mapstructure:"key_name" json:"key_name,omitempty" bson:"key_name,omitempty"`

	// MountPoints are the disk mount points for EBS volumes.
	MountPoints []MountPoint `mapstructure:"mount_points" json:"mount_points,omitempty" bson:"mount_points,omitempty"`

	// SecurityGroupIDs is a list of security group IDs.
	SecurityGroupIDs []string `mapstructure:"security_group_ids" json:"security_group_ids,omitempty" bson:"security_group_ids,omitempty"`

	// SubnetId is only set in a VPC. Either subnet id or vpc name must set.
	SubnetId string `mapstructure:"subnet_id" json:"subnet_id,omitempty" bson:"subnet_id,omitempty"`

	// VpcName is used to get the subnet ID automatically. Either subnet id or vpc name must set.
	VpcName string `mapstructure:"vpc_name" json:"vpc_name,omitempty" bson:"vpc_name,omitempty"`

	// IsVpc is set to true if the security group is part of a VPC.
	IsVpc bool `mapstructure:"is_vpc" json:"is_vpc,omitempty" bson:"is_vpc,omitempty"`

	// BidPrice is the price we are willing to pay for a spot instance.
	BidPrice float64 `mapstructure:"bid_price" json:"bid_price,omitempty" bson:"bid_price,omitempty"`

	// UserData are commands to run after the instance starts.
	UserData string `mapstructure:"user_data" json:"user_data" bson:"user_data,omitempty"`

	// Region is the EC2 region in which the instance will start. If empty,
	// the ec2Manager will spawn in "us-east-1".
	Region string `mapstructure:"region" json:"region" bson:"region,omitempty"`
}

EC2ProviderSettings describes properties of managed instances.

func (*EC2ProviderSettings) Validate 

func (s *EC2ProviderSettings) Validate() error

Validate that essential EC2ProviderSettings fields are not empty.

type GCESettings 

type GCESettings struct {
	Project string `mapstructure:"project_id"`
	Zone    string `mapstructure:"zone"`

	ImageName   string `mapstructure:"image_name"`
	ImageFamily string `mapstructure:"image_family"`

	MachineName string `mapstructure:"instance_type"`
	NumCPUs     int64  `mapstructure:"num_cpus"`
	MemoryMB    int64  `mapstructure:"memory_mb"`

	DiskType   string `mapstructure:"disk_type"`
	DiskSizeGB int64  `mapstructure:"disk_size_gb"`

	// Network tags are used to configure network firewalls.
	NetworkTags []string `mapstructure:"network_tags"`

	// By default, GCE uses project-wide SSH keys. Project-wide keys should be manually
	// added to the project metadata. These SSH keys are optional instance-wide keys.
	SSHKeys sshKeyGroup `mapstructure:"ssh_keys"`
}

GCESettings specifies the settings used to configure a host instance.

func (*GCESettings) Validate 

func (opts *GCESettings) Validate() error

Validate verifies a set of GCESettings.

type Manager 

type Manager interface {
	// Returns a pointer to the manager's configuration settings struct
	GetSettings() ProviderSettings

	//Load credentials or other settings from the config file
	Configure(context.Context, *evergreen.Settings) error

	// SpawnHost attempts to create a new host by requesting one from the
	// provider's API.
	SpawnHost(context.Context, *host.Host) (*host.Host, error)

	// ModifyHost modifies an existing host
	ModifyHost(context.Context, *host.Host, host.HostModifyOptions) error

	// get the status of an instance
	GetInstanceStatus(context.Context, *host.Host) (CloudStatus, error)

	// TerminateInstances destroys the host in the underlying provider
	TerminateInstance(context.Context, *host.Host, string) error

	//IsUp returns true if the underlying provider has not destroyed the
	//host (in other words, if the host "should" be reachable. This does not
	//necessarily mean that the host actually *is* reachable via SSH
	IsUp(context.Context, *host.Host) (bool, error)

	//Called by the hostinit process when the host is actually up. Used
	//to set additional provider-specific metadata
	OnUp(context.Context, *host.Host) error

	// GetDNSName returns the DNS name of a host.
	GetDNSName(context.Context, *host.Host) (string, error)

	// GetSSHOptions generates the command line args to be passed to ssh to
	// allow connection to the machine
	GetSSHOptions(*host.Host, string) ([]string, error)

	// TimeTilNextPayment returns how long there is until the next payment
	// is due for a particular host
	TimeTilNextPayment(*host.Host) time.Duration
}

Manager is an interface which handles creating new hosts or modifying them via some third-party API.

func GetManager 

func GetManager(ctx context.Context, mgrOpts ManagerOpts, settings *evergreen.Settings) (Manager, error)

GetManager returns an implementation of Manager for the given manager options. It returns an error if the provider name doesn't have a known implementation.

func NewEC2FleetManager 

func NewEC2FleetManager(opts *EC2FleetManagerOptions) Manager

func NewEC2Manager 

func NewEC2Manager(opts *EC2ManagerOptions) Manager

NewEC2Manager creates a new manager of EC2 spot and on-demand instances.

type ManagerOpts 

type ManagerOpts struct {
	Provider string
	Region   string
}

ManagerOpts is a struct containing the fields needed to get a new cloud manager of the proper type.

type MockInstance 

type MockInstance struct {
	IsUp               bool
	IsSSHReachable     bool
	Status             CloudStatus
	SSHOptions         []string
	TimeTilNextPayment time.Duration
	DNSName            string
	OnUpRan            bool
}

MockInstance mocks a running server that Evergreen knows about. It contains fields that can be set to change the response the cloud manager returns when this mock instance is queried for.

type MockProvider 

type MockProvider interface {
	Len() int
	Reset()
	Get(string) MockInstance
	Set(string, MockInstance)
	IterIDs() <-chan string
	IterInstances() <-chan MockInstance
}

func GetMockProvider 

func GetMockProvider() MockProvider

type MockProviderSettings 

type MockProviderSettings struct {
	Region string `mapstructure:"region" json:"region" bson:"region,omitempty"`
}

type MountPoint 

type MountPoint struct {
	VirtualName string `mapstructure:"virtual_name" json:"virtual_name,omitempty" bson:"virtual_name,omitempty"`
	DeviceName  string `mapstructure:"device_name" json:"device_name,omitempty" bson:"device_name,omitempty"`
	Size        int64  `mapstructure:"size" json:"size,omitempty" bson:"size,omitempty"`
	Iops        int64  `mapstructure:"iops" json:"iops,omitempty" bson:"iops,omitempty"`
	SnapshotID  string `mapstructure:"snapshot_id" json:"snapshot_id,omitempty" bson:"snapshot_id,omitempty"`
	VolumeType  string `mapstructure:"volume_type" json:"volume_type,omitempty" bson:"volume_type,omitempty"`
}

type ProviderSettings 

type ProviderSettings interface {
	Validate() error
}

ProviderSettings exposes provider-specific configuration settings for a Manager.

type SpawnOptions 

type SpawnOptions struct {
	DistroId         string
	ProviderSettings *map[string]interface{}
	UserName         string
	PublicKey        string
	TaskId           string
	Owner            *user.DBUser
	InstanceTags     []host.Tag
}

Options holds the required parameters for spawning a host.

type StaticHost 

type StaticHost struct {
	Name string `mapstructure:"name" json:"name" bson:"name"`
}

type StaticSettings 

type StaticSettings struct {
	Hosts []StaticHost `mapstructure:"hosts" json:"hosts" bson:"hosts"`
}

func (*StaticSettings) Validate 

func (s *StaticSettings) Validate() error

Validate checks that the settings from the configuration are valid.

type Terms 

type Terms struct {
	OnDemand map[string]map[string]struct {
		PriceDimensions map[string]struct {
			PricePerUnit struct {
				USD string
			}
		}
	}
}

Terms is an internal type for loading price API results into.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.