common

package
v0.9.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 6, 2018 License: Apache-2.0 Imports: 1 Imported by: 96

Documentation

Index

Constants

View Source 
const (
	// The supported catalog services
	CatalogService_MongoDB       = "mongodb"
	CatalogService_PostgreSQL    = "postgresql"
	CatalogService_Cassandra     = "cassandra"
	CatalogService_ZooKeeper     = "zookeeper"
	CatalogService_Kafka         = "kafka"
	CatalogService_KafkaManager  = "kafkamanager"
	CatalogService_KafkaSinkES   = "kafkasinkes"
	CatalogService_Redis         = "redis"
	CatalogService_CouchDB       = "couchdb"
	CatalogService_Consul        = "consul"
	CatalogService_ElasticSearch = "elasticsearch"
	CatalogService_Kibana        = "kibana"
	CatalogService_Logstash      = "logstash"
	CatalogService_Telegraf      = "telegraf"

	// The status of one service
	// ServiceStatusCreating: creating the required resources of the service, such as volumes.
	ServiceStatusCreating = "CREATING"
	// ServiceStatusInitializing: service resources are created, wait for the initialization.
	// at this state, the service containers are running. Some services require the special
	// initialization configurations, such as configuring the DB replicaset. Some services
	// does not require this step, such as Cassandra. The service creation script could simply
	// change the service status to ACTIVE.
	ServiceStatusInitializing = "INITIALIZING"
	// ServiceStatusActive: service initialized and ready to use.
	ServiceStatusActive   = "ACTIVE"
	ServiceStatusDeleting = "DELETING"
	ServiceStatusDeleted  = "DELETED"

	// Service member status.
	ServiceMemberStatusActive = "ACTIVE"
	ServiceMemberStatusPause  = "PAUSE"
	ServiceMemberStatusBad    = "Bad"

	AnyAvailabilityZone = "any"

	// The status of one task
	TaskStatusRunning = "RUNNING"
	TaskStatusStopped = "STOPPED"

	// Task types
	TaskTypeInit = "init"

	// The journal volume name prefix, the journal volume name will be journal-serviceuuid,
	// the mount path will be /mnt/journal-serviceuuid
	JournalVolumeNamePrefix = "journal"

	// General Purpose SSD
	VolumeTypeGPSSD = "gp2"
	// Provisioned IOPS SSD
	VolumeTypeIOPSSSD = "io1"
	// Throughput Optimized HDD
	VolumeTypeTPHDD = "st1"

	ServiceNamePattern = "[a-zA-Z][a-zA-Z0-9-]*"
)
View Source 
const (
	Version    = "0.9.5"
	Version095 = "0.9.5"

	CloudPlatformAWS = "aws"

	ContainerPlatformECS   = "ecs"
	ContainerPlatformSwarm = "swarm"
	ContainerPlatformK8s   = "k8s"

	DefaultK8sNamespace = "default"

	ContainerPlatformRoleManager = "manager"
	ContainerPlatformRoleWorker  = "worker"

	// OrgName and SystemName could not include "-"
	OrgName             = "cloudstax/"
	SystemName          = "firecamp"
	ContainerNamePrefix = OrgName + SystemName + "-"

	DefaultFSType = "xfs"

	// VolumeDriverName is the name for docker volume driver.
	// Do NOT change the volume driver name. If this name is changed,
	// please update the volume driver plugin in scripts/builddocker.sh.
	// please also update docker/volume/aws-ecs-agent-patch/firecamp_task_engine.go,
	// Has the separate definition in firecamp_task_engine.go aims to avoid the dependency of
	// ecs-agent on firecamp code.
	VolumeDriverName = OrgName + SystemName + "-" + "volume:" + Version

	// LogDriverName is the name for docker log driver.
	// Do NOT change the log driver name. If this name is changed,
	// please update the log driver plugin in scripts/builddocker.sh.
	LogDriverName = OrgName + SystemName + "-" + "log:" + Version
	// The LogServiceUUIDKey is set by firecamp_task_engine.go in cloudstax/amazon-ecs-agent.
	// if you want to change the value here, also need to change in cloudstax/amazon-ecs-agent.
	LogServiceUUIDKey = "ServiceUUID"
	// The LogServiceMemberKey is for the single member service, such as the management service.
	LogServiceMemberKey = "ServiceMember"

	LOGDRIVER_DEFAULT = "json-file"
	LOGDRIVER_AWSLOGS = "awslogs"

	DefaultLogDir = "/var/log/" + SystemName

	NameSeparator = "-"

	ServiceMemberDomainNameTTLSeconds = 5

	// A ECS container instance has 1,024 cpu units for every CPU core
	DefaultMaxCPUUnits     = 0
	DefaultReserveCPUUnits = 256
	DefaultMaxMemoryMB     = 0
	DefaultReserveMemoryMB = 256

	DefaultServiceWaitSeconds = 120
	DefaultTaskWaitSeconds    = 120
	DefaultTaskRetryCounts    = 5
	DefaultRetryWaitSeconds   = 3
	CliRetryWaitSeconds       = 5

	DomainSeparator  = "."
	DomainNameSuffix = SystemName
	DomainCom        = "com"

	DefaultHostIP = "127.0.0.1"

	ContainerNameSuffix = "container"

	DefaultContainerMountPath              = "/data"
	DefaultJournalVolumeContainerMountPath = "/journal"
	DefaultConfigDir                       = "/conf"
	DefaultConfigPath                      = DefaultContainerMountPath + DefaultConfigDir
	DefaultConfigFileMode                  = 0600

	DBTypeControlDB = "controldb" // the controldb service
	DBTypeCloudDB   = "clouddb"   // such as AWS DynamoDB
	DBTypeK8sDB     = "k8sdb"     // db on top of k8s ConfigMap
	DBTypeMemDB     = "memdb"     // in-memory db, for test only

	ControlDBServerPort  = 27030
	ControlDBName        = "controldb"
	ControlDBServiceName = SystemName + NameSeparator + ControlDBName
	ControlDBDefaultDir  = DefaultContainerMountPath + "/" + ControlDBName
	// ControlDBUUIDPrefix defines the prefix of the controldb server id.
	// The service uuid of the controldb service would be ControlDBUUIDPrefix + volumeID.
	// The volumeID is the ID of the volume created for the controldb service.
	ControlDBUUIDPrefix      = ControlDBName + NameSeparator
	ControlDBContainerImage  = OrgName + ControlDBServiceName + ":" + Version
	ControlDBReserveCPUUnits = 256
	ControlDBMaxMemMB        = 4096
	ControlDBReserveMemMB    = 256
	ControlDBVolumeSizeGB    = int64(4)

	ManageHTTPServerPort  = 27040
	ManageName            = "manageserver"
	ManageServiceName     = SystemName + NameSeparator + ManageName
	ManageContainerImage  = OrgName + ManageServiceName + ":" + Version
	ManageReserveCPUUnits = 256
	ManageMaxMemMB        = 4096
	ManageReserveMemMB    = 256

	ServiceTypeStateful  = "stateful"
	ServiceTypeStateless = "stateless"

	// The service DNS name will be ServiceName-index.ClusterName-firecamp.com.
	// Each label of the domain name can be up to 63 bytes long. Reserve 4 bytes for the index.
	// The ServiceName can be up to 58 bytes.
	MaxServiceNameLength = 58
)
View Source 
const (
	// this is for passing the firecamp version to AWS ECS task definition.
	// the ECS agent patch will construct the correct volume plugin name with it.
	ENV_VERSION = "VERSION"

	ENV_VALUE_SEPARATOR = ","
	ENV_SHARD_SEPARATOR = ";"

	ENV_REGION            = "REGION"
	ENV_CLUSTER           = "CLUSTER"
	ENV_MANAGE_SERVER_URL = "MANAGE_SERVER_URL"
	ENV_OP                = "OP"

	ENV_ADMIN          = "ADMIN"
	ENV_ADMIN_PASSWORD = "ADMIN_PASSWORD"

	ENV_SERVICE_NAME    = "SERVICE_NAME"
	ENV_SERVICE_NODE    = "SERVICE_NODE"
	ENV_SERVICE_PORT    = "SERVICE_PORT"
	ENV_SERVICE_MASTER  = "SERVICE_MASTER"
	ENV_SERVICE_MEMBERS = "SERVICE_MEMBERS"
	ENV_SERVICE_TYPE    = "SERVICE_TYPE"

	ENV_CONTAINER_PLATFORM = "CONTAINER_PLATFORM"
	ENV_DB_TYPE            = "DB_TYPE"
	ENV_AVAILABILITY_ZONES = "AVAILABILITY_ZONES"
	ENV_K8S_NAMESPACE      = "K8S_NAMESPACE"

	ENV_SHARDS            = "SHARDS"
	ENV_REPLICAS_PERSHARD = "REPLICAS_PERSHARD"
)

define the common environment keys

Variables

View Source 
var (
	ErrInternal               = errors.New("InternalError")
	ErrTimeout                = errors.New("Timeout")
	ErrSystemCreating         = errors.New("System tables are at the creating status, please retry later")
	ErrServiceExist           = errors.New("Service exists")
	ErrServiceDeleting        = errors.New("Service deleting")
	ErrServiceDeleted         = errors.New("Service deleted")
	ErrConfigMismatch         = errors.New("Config mismatch")
	ErrInvalidArgs            = errors.New("InvalidArgs")
	ErrUnsupportedPlatform    = errors.New("Not supported container platform")
	ErrNotFound               = errors.New("NotFound")
	ErrNotSupported           = errors.New("NotSupported")
	ErrConditionalCheckFailed = errors.New("ConditionalCheckFailed")
)

Functions

This section is empty.

Types

type CasUserAttr added in v0.9.2 

type CasUserAttr struct {
	HeapSizeMB      int64
	JmxRemoteUser   string
	JmxRemotePasswd string
}

CasUserAttr represents the custom Cassandra service attributes.

type ConfigFile 

type ConfigFile struct {
	ServiceUUID  string // partition key
	FileID       string // sort key
	FileMD5      string
	FileName     string
	FileMode     uint32
	LastModified int64
	Content      string // The content of the config file.
}

ConfigFile represents the detail config content of service member. To update the ConfigFile content of one replica, 2 steps are required: 1) create a new ConfigFile with a new FileID, 2) update ServiceMember MemberConfig to point to the new ConfigFile. We could not directly update the old ConfigFile. If node crashes before step2, ServiceMember MemberConfig will not be consistent with ConfigFile.

type ConsulUserAttr added in v0.9.2 

type ConsulUserAttr struct {
	Datacenter string
	Domain     string

	Encrypt   string
	EnableTLS bool
	HTTPSPort int64
}

ConsulUserAttr represents the consul service attributes.

type CouchDBUserAttr added in v0.9.2 

type CouchDBUserAttr struct {
	Admin           string
	EncryptedPasswd string

	// CouchDB Cors configs
	EnableCors  bool
	Credentials bool
	Origins     string
	Headers     string
	Methods     string

	EnableSSL bool
}

CouchDBUserAttr represents the couchdb service attributes.

type Device 

type Device struct {
	ClusterName string // partition key
	DeviceName  string // sort key
	// service that the device is assigned to
	ServiceName string
}

Device records the assigned device for the service. The DeviceName has to be part of the key, to ensure one device is only assigned to one service.

type ESUserAttr added in v0.9.2 

type ESUserAttr struct {
	HeapSizeMB             int64
	DedicatedMasters       int64
	DisableDedicatedMaster bool
	DisableForceAwareness  bool
	// DataNodes are the number of ElasticSearch data nodes, this field is added in 0.9.5.
	// Assume no one is using ElasticSearch before 0.9.5. No upgrade is supported.
	DataNodes int64
}

ESUserAttr represents the elasticsearch service attributes.

type EnvKeyValuePair 

type EnvKeyValuePair struct {
	Name  string
	Value string
}

type KCSinkESUserAttr added in v0.9.5 

type KCSinkESUserAttr struct {
	// Kafka Connect JVM heap size
	HeapSizeMB int64

	// The Kafka service to sink from
	KafkaServiceName string

	// The Kafka Topic to read data from
	Topic string

	// The ElasticSearch service to sink to
	ESServiceName string

	// The type to use for each index
	TypeName string

	MaxBufferedRecords int
	BatchSize          int

	// The storage replication factor for storage, offset and status topics
	ConfigReplFactor uint
	OffsetReplFactor uint
	StatusReplFactor uint
}

KCSinkESUserAttr represents the kafka connect service attributes.

type KMUserAttr added in v0.9.5 

type KMUserAttr struct {
	// Kafka Manager JVM heap size
	HeapSizeMB int64

	// Kafka Manager user and password
	User     string
	Password string

	// The existing ZooKeeper service that Kafka Manager will use.
	ZkServiceName string
}

KMUserAttr represents the kafka manager service attributes. This is added in 0.9.5, upgrade to 0.9.5 will create the user attr for the existing kafka manager service.

type KafkaUserAttr added in v0.9.2 

type KafkaUserAttr struct {
	HeapSizeMB     int64
	AllowTopicDel  bool
	RetentionHours int64
	ZkServiceName  string
	// jmx remote user is added in 0.9.5, upgrade to 0.9.5 will do:
	// - create jmx password and access file
	// - add jmx options into java env file
	// - add jmx user and password to KafkaUserAttr
	// - update service spec to expose the jmx listening port
	JmxRemoteUser   string
	JmxRemotePasswd string
}

KafkaUserAttr represents the kafka service attributes.

type KeyValuePair added in v0.9.1 

type KeyValuePair struct {
	Key   string
	Value string
}

type KibanaUserAttr added in v0.9.2 

type KibanaUserAttr struct {
	ESServiceName string
	ProxyBasePath string
	EnableSSL     bool
}

KibanaUserAttr represents the kibana service attributes.

type LSUserAttr added in v0.9.2 

type LSUserAttr struct {
	HeapSizeMB     int64
	ContainerImage string

	QueueType             string
	EnableDeadLetterQueue bool

	PipelineConfigs       string
	PipelineWorkers       int
	PipelineOutputWorkers int
	PipelineBatchSize     int
	PipelineBatchDelay    int
}

LSUserAttr represents the logstash service attributes.

type MemberConfig 

type MemberConfig struct {
	FileName string
	// The config file uuid
	FileID string
	// The MD5 checksum of the config file content.
	// The config file content would usually not be updated. The checksum could help
	// the volume driver easily know whether it needs to load the config file content.
	// The config file content may not be small, such as a few hundreds KB.
	FileMD5 string
}

MemberConfig represents the configs of one member

type MemberVolumes added in v0.9.1 

type MemberVolumes struct {
	// The config files will be created on the primary volume.
	PrimaryVolumeID string
	// The primary device will be mounted to /mnt/serviceuuid on the host and /data in the container
	PrimaryDeviceName string

	// The possible journal volume to store the service's journal
	JournalVolumeID string
	// The journal device will be mounted to /mnt/journal-serviceuuid on the host and /journal in the container.
	// We don't want to mount the journal device under the primary device mount path, such as /mnt/serviceuuid/journal.
	// If there is some bug that the journal device is not mounted, the service journal will be directly written to
	// the primary device. Later when the journal device is mounted again, some journal will be temporary lost.
	JournalDeviceName string
}

MemberVolumes represent the volumes of one member.

type MongoDBUserAttr added in v0.9.1 

type MongoDBUserAttr struct {
	// if ReplicaSetOnly == true and Shards == 1, create a single replicaset, else create a sharded cluster.
	Shards           int64
	ReplicasPerShard int64
	ReplicaSetOnly   bool
	// the number of config servers, ignored if ReplicaSetOnly == true and Shards == 1.
	ConfigServers int64
	// the content of the key file.
	KeyFileContent string
}

MongoDBUserAttr represents the custom MongoDB service attributes.

type PortMapping 

type PortMapping struct {
	ContainerPort int64
	HostPort      int64
	// whether the port is a service port, used by k8s headless service for statefulset.
	IsServicePort bool
}

PortMapping defines the container port to host port mapping.

type PostgresUserAttr added in v0.9.2 

type PostgresUserAttr struct {
	ContainerImage string
}

PostgresUserAttr represents the postgresql service attributes.

type RedisUserAttr added in v0.9.1 

type RedisUserAttr struct {
	Shards           int64
	ReplicasPerShard int64

	MemoryCacheSizeMB int64
	DisableAOF        bool
	AuthPass          string
	ReplTimeoutSecs   int64
	MaxMemPolicy      string
	ConfigCmdName     string
}

RedisUserAttr represents the custom Redis service attributes.

type Resources 

type Resources struct {
	// The number of cpu units for the container.
	// A container instance has 1,024 cpu units for every CPU core.
	// MaxCPUUnits specifies how much of the available CPU resources a container can use.
	// This will set docker's --cpu-period and --cpu-quota.
	// ReserveCPUUnits is docker's --cpu-share, same with the ECS cpu units.
	// The value could be -1, which means no limit on the resource.
	MaxCPUUnits     int64
	ReserveCPUUnits int64
	MaxMemMB        int64
	ReserveMemMB    int64
}

Resources represents the service/task resources, cpu and memory.

type Service 

type Service struct {
	ClusterName string // partition key
	ServiceName string // sort key
	ServiceUUID string
}

Service records the assigned uuid for the service.

type ServiceAttr 

type ServiceAttr struct {
	ServiceUUID   string // partition key
	ServiceStatus string
	LastModified  int64
	Replicas      int64
	ClusterName   string
	ServiceName   string
	Volumes       ServiceVolumes

	// whether the service members need to know each other, such as database replicas.
	// if yes, the member will be registered to DNS. in aws, DNS will be Route53.
	RegisterDNS bool
	// for v1, DomainName would be the default domain, such as cluster-firecamp.com
	DomainName string
	// The AWS Route53 HostedZone for the current firecamp cluster.
	HostedZoneID string

	// Whether the service member needs the static ip. This is required by Redis & Consul.
	RequireStaticIP bool

	// The custom service attributes.
	// This field should not be nil. Every service will have its own user attr.
	UserAttr *ServiceUserAttr

	// The specified resources for the service.
	Resource Resources

	// ServiceType is added after release 0.9.3, to support stateless service such as kafka manager.
	ServiceType string
}

ServiceAttr represents the detail attributes of the service. The ServiceUUID is passed as volume name to the volume driver. The volume driver could get the detail service attributes for other operations.

type ServiceMember 

type ServiceMember struct {
	ServiceUUID string // partition key
	MemberIndex int64  // sort key
	// The member status: Active, Pause, Bad.
	// This will be useful for some cases. For example, prevent the member container
	// from running when doing some maintenance for one member.
	Status string
	// The service member name, such as mypg-0, myredis-0. MemberName.DomainName is the member's DNS name.
	MemberName string
	// For the stateless service, AvailableZone is ingored, simply set to "any".
	AvailableZone       string
	TaskID              string
	ContainerInstanceID string
	ServerInstanceID    string
	LastModified        int64

	// The volumes of one member. One member could have multiple volumes.
	// For example, one for DB data, the other for journal.
	// For the stateless service that does not require volume, it could be empty.
	Volumes MemberVolumes

	// The static IP assigned to this member
	StaticIP string

	// One member could have multiple config files.
	// For example, cassandra.yaml and rackdc properties files.
	Configs []*MemberConfig
}

ServiceMember represents the attributes of one service member.

type ServiceStaticIP added in v0.8.1 

type ServiceStaticIP struct {
	StaticIP string // partition key

	// The owner service of this static IP.
	ServiceUUID string
	// TODO adding MemberName would be a good optimization.
	//      so volume plugin could directly get the local static IP and get the member.
	// MemberName  string
	// The AvailableZone this static IP belongs to.
	AvailableZone string
	// The server instance this IP is assigned to.
	ServerInstanceID string
	// The network interface this IP is assigned to.
	NetworkInterfaceID string
}

ServiceStaticIP represents the owner service of one static IP.

type ServiceStatus 

type ServiceStatus struct {
	RunningCount int64
	DesiredCount int64
}

ServiceStatus represents the service's running status. TODO add more status. For example, the members are running on which nodes.

type ServiceUserAttr added in v0.9.1 

type ServiceUserAttr struct {
	// catalog service type such as CatalogService_MongoDB.
	ServiceType string
	AttrBytes   []byte
}

ServiceUserAttr represents the custom service attributes.

type ServiceVolume added in v0.9.1 

type ServiceVolume struct {
	VolumeType   string
	VolumeSizeGB int64
	Iops         int64
	Encrypted    bool
}

ServiceVolume contains the volume parameters.

type ServiceVolumes added in v0.9.1 

type ServiceVolumes struct {
	PrimaryDeviceName string
	PrimaryVolume     ServiceVolume

	// The JournalDeviceName could be empty if service only needs one volume.
	JournalDeviceName string
	JournalVolume     ServiceVolume
}

ServiceVolumes represent the volumes of one service. For now, allow maximum 2 devices. Could further expand if necessary in the future.

type TGUserAttr added in v0.9.5 

type TGUserAttr struct {
	CollectIntervalSecs int
	MonitorServiceName  string
	MonitorMetrics      string
}

TGUserAttr represents the telegraf service attributes.

type TaskStatus 

type TaskStatus struct {
	// valid status: pending, running, stopped
	Status        string
	StoppedReason string
	StartedAt     string // time format: 2017-05-20T19:50:56.834924582Z
	FinishedAt    string
}

TaskStatus represents the task's status

type ZKUserAttr added in v0.9.2 

type ZKUserAttr struct {
	HeapSizeMB int64
	// jmx remote user is added in 0.9.5, upgrade to 0.9.5 will do:
	// - create jmx password and access file
	// - add jmx options into java env file
	// - add jmx user and password to ZKUserAttr
	// - update service spec to expose the jmx listening port
	JmxRemoteUser   string
	JmxRemotePasswd string
}

ZKUserAttr represents the zookeeper service attributes.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.