assets

package
v1.10.0-rc3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 3, 2020 License: Apache-2.0 Imports: 19 Imported by: 17

Documentation

Index

Constants

View Source 
const (
	// DefaultRequireCriticalServersOnly is the default for requiring critical servers only.
	// (bryce) this default is set here and in the service env config, need to figure out how to refactor
	// this to be in one place.
	DefaultRequireCriticalServersOnly = false
)
View Source 
const (
	// DefaultUploadConcurrencyLimit is the default maximum number of concurrent object storage uploads.
	// (bryce) this default is set here and in the service env config, need to figure out how to refactor
	// this to be in one place.
	DefaultUploadConcurrencyLimit = 100
)
View Source 
const (
	// RequireCriticalServersOnlyEnvVar is the environment variable for requiring critical servers only.
	RequireCriticalServersOnlyEnvVar = "REQUIRE_CRITICAL_SERVERS_ONLY"
)
View Source 
const (
	// UploadConcurrencyLimitEnvVar is the environment variable for the upload concurrency limit.
	UploadConcurrencyLimitEnvVar = "STORAGE_UPLOAD_CONCURRENCY_LIMIT"
)

Variables

View Source 
var (

	// ServiceAccountName is the name of Pachyderm's service account.
	// It's public because it's needed by pps.APIServer to create the RCs for
	// workers.
	ServiceAccountName = "pachyderm"

	// PrometheusPort hosts the prometheus stats for scraping
	PrometheusPort = 656

	// IAMAnnotation is the annotation used for the IAM role, this can work
	// with something like kube2iam as an alternative way to provide
	// credentials.
	IAMAnnotation = "iam.amazonaws.com/role"
)

Functions

func AddRegistry added in v1.6.1 

func AddRegistry(registry string, imageName string) string

AddRegistry switches the registry that an image is targeting, unless registry is blank

func AmazonIAMRoleSecret added in v1.7.2 

func AmazonIAMRoleSecret(region, bucket, distribution string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonIAMRoleSecret creates an amazon secret with the following parameters: 

region         - AWS region
bucket         - S3 bucket name
distribution   - cloudfront distribution
advancedConfig - advanced configuration

func AmazonSecret 

func AmazonSecret(region, bucket, id, secret, token, distribution, endpoint string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonSecret creates an amazon secret with the following parameters: 

region         - AWS region
bucket         - S3 bucket name
id             - AWS access key id
secret         - AWS secret access key
token          - AWS access token
distribution   - cloudfront distribution
endpoint       - Custom endpoint (generally used for S3 compatible object stores)
advancedConfig - advanced configuration

func AmazonVaultSecret added in v1.7.1 

func AmazonVaultSecret(region, bucket, vaultAddress, vaultRole, vaultToken, distribution string, advancedConfig *obj.AmazonAdvancedConfiguration) map[string][]byte

AmazonVaultSecret creates an amazon secret with the following parameters: 

region         - AWS region
bucket         - S3 bucket name
vaultAddress   - address/hostport of vault
vaultRole      - pachd's role in vault
vaultToken     - pachd's vault token
distribution   - cloudfront distribution
advancedConfig - advanced configuration

func ClusterRole added in v1.6.7 

func ClusterRole(opts *AssetOpts) *rbacv1.ClusterRole

ClusterRole returns a ClusterRole that should be bound to the Pachyderm service account.

func ClusterRoleBinding added in v1.6.7 

func ClusterRoleBinding(opts *AssetOpts) *rbacv1.ClusterRoleBinding

ClusterRoleBinding returns a ClusterRoleBinding that binds Pachyderm's ClusterRole to its ServiceAccount.

func DashDeployment added in v1.4.5 

func DashDeployment(opts *AssetOpts) *apps.Deployment

DashDeployment creates a Deployment for the pachyderm dashboard.

func DashService added in v1.4.5 

func DashService(opts *AssetOpts) *v1.Service

DashService creates a Service for the pachyderm dashboard.

func EtcdDeployment added in v1.4.5 

func EtcdDeployment(opts *AssetOpts, hostPath string) *apps.Deployment

EtcdDeployment returns an etcd k8s Deployment.

func EtcdHeadlessService added in v1.3.19 

func EtcdHeadlessService(opts *AssetOpts) *v1.Service

EtcdHeadlessService returns a headless etcd service, which is only for DNS resolution.

func EtcdNodePortService added in v1.3.19 

func EtcdNodePortService(local bool, opts *AssetOpts) *v1.Service

EtcdNodePortService returns a NodePort etcd service. This will let non-etcd pods talk to etcd

func EtcdStatefulSet added in v1.3.19 

func EtcdStatefulSet(opts *AssetOpts, backend backend, diskSpace int) interface{}

EtcdStatefulSet returns a stateful set that manages an etcd cluster

func EtcdStorageClass added in v1.3.19 

func EtcdStorageClass(opts *AssetOpts, backend backend) (interface{}, error)

EtcdStorageClass creates a storage class used for dynamic volume provisioning. Currently dynamic volume provisioning only works on AWS and GCE.

func EtcdVolume added in v1.3.19 

func EtcdVolume(persistentDiskBackend backend, opts *AssetOpts,
	hostPath string, name string, size int) (*v1.PersistentVolume, error)

EtcdVolume creates a persistent volume backed by a volume with name "name"

func EtcdVolumeClaim added in v1.3.19 

func EtcdVolumeClaim(size int, opts *AssetOpts) *v1.PersistentVolumeClaim

EtcdVolumeClaim creates a persistent volume claim of 'size' GB.

Note that if you're controlling Etcd with a Stateful Set, this is unnecessary (the stateful set controller will create PVCs automatically).

func GetBackendSecretVolumeAndMount added in v1.7.4 

func GetBackendSecretVolumeAndMount(backend string) (v1.Volume, v1.VolumeMount)

GetBackendSecretVolumeAndMount returns a properly configured Volume and VolumeMount object given a backend. The backend needs to be one of the constants defined in pfs/server.

func GetSecretEnvVars added in v1.8.0 

func GetSecretEnvVars(storageBackend string) []v1.EnvVar

GetSecretEnvVars returns the environment variable specs for the storage secret.

func GithookService added in v1.6.7 

func GithookService(namespace string) *v1.Service

GithookService returns a k8s service that exposes a public IP

func GoogleSecret 

func GoogleSecret(bucket string, cred string) map[string][]byte

GoogleSecret creates a google secret with a bucket name.

func Images added in v1.6.1 

func Images(opts *AssetOpts) []string

Images returns a list of all the images that are used by a pachyderm deployment.

func LocalSecret added in v1.6.0 

func LocalSecret() map[string][]byte

LocalSecret creates an empty secret.

func MicrosoftSecret added in v1.2.3 

func MicrosoftSecret(container string, id string, secret string) map[string][]byte

MicrosoftSecret creates a microsoft secret with following parameters: 

container - Azure blob container
id    	   - Azure storage account name
secret    - Azure storage account key

func MinioSecret added in v1.3.5 

func MinioSecret(bucket string, id string, secret string, endpoint string, secure, isS3V2 bool) map[string][]byte

MinioSecret creates an amazon secret with the following parameters: 

bucket - S3 bucket name
id     - S3 access key id
secret - S3 secret access key
endpoint  - S3 compatible endpoint
secure - set to true for a secure connection.
isS3V2 - Set to true if client follows S3V2

func PachdDeployment added in v1.4.5 

func PachdDeployment(opts *AssetOpts, objectStoreBackend backend, hostPath string) *apps.Deployment

PachdDeployment returns a pachd k8s Deployment.

func PachdPeerService 

func PachdPeerService(opts *AssetOpts) *v1.Service

PachdPeerService returns an internal pachd service. This service will reference the PeerPorr, which does not employ TLS even if cluster TLS is enabled. Because of this, the service is a `ClusterIP` type (i.e. not exposed outside of the cluster.)

func PachdService 

func PachdService(opts *AssetOpts) *v1.Service

PachdService returns a pachd service.

func Role added in v1.7.4 

func Role(opts *AssetOpts) *rbacv1.Role

Role returns a Role that should be bound to the Pachyderm service account.

func RoleBinding added in v1.7.4 

func RoleBinding(opts *AssetOpts) *rbacv1.RoleBinding

RoleBinding returns a RoleBinding that binds Pachyderm's Role to its ServiceAccount.

func ServiceAccount 

func ServiceAccount(opts *AssetOpts) *v1.ServiceAccount

ServiceAccount returns a kubernetes service account for use with Pachyderm.

func WriteAmazonAssets 

func WriteAmazonAssets(encoder serde.Encoder, opts *AssetOpts, region string, bucket string, volumeSize int, creds *AmazonCreds, cloudfrontDistro string, advancedConfig *obj.AmazonAdvancedConfiguration) error

WriteAmazonAssets writes assets to an amazon backend.

func WriteAssets 

func WriteAssets(encoder serde.Encoder, opts *AssetOpts, objectStoreBackend backend,
	persistentDiskBackend backend, volumeSize int,
	hostPath string) error

WriteAssets writes the assets to encoder.

func WriteCustomAssets added in v1.3.12 

func WriteCustomAssets(encoder serde.Encoder, opts *AssetOpts, args []string, objectStoreBackend string,
	persistentDiskBackend string, secure, isS3V2 bool, advancedConfig *obj.AmazonAdvancedConfiguration) error

WriteCustomAssets writes assets to a custom combination of object-store and persistent disk.

func WriteDashboardAssets added in v1.4.5 

func WriteDashboardAssets(encoder serde.Encoder, opts *AssetOpts) error

WriteDashboardAssets writes the k8s config for deploying the Pachyderm dashboard to 'encoder'

func WriteGoogleAssets 

func WriteGoogleAssets(encoder serde.Encoder, opts *AssetOpts, bucket string, cred string, volumeSize int) error

WriteGoogleAssets writes assets to a google backend.

func WriteLocalAssets 

func WriteLocalAssets(encoder serde.Encoder, opts *AssetOpts, hostPath string) error

WriteLocalAssets writes assets to a local backend.

func WriteMicrosoftAssets added in v1.2.3 

func WriteMicrosoftAssets(encoder serde.Encoder, opts *AssetOpts, container string, id string, secret string, volumeSize int) error

WriteMicrosoftAssets writes assets to a microsoft backend

func WriteSecret added in v1.6.0 

func WriteSecret(encoder serde.Encoder, data map[string][]byte, opts *AssetOpts) error

WriteSecret writes a JSON-encoded k8s secret to the given writer. The secret uses the given map as data.

func WriteTLSSecret added in v1.7.4 

func WriteTLSSecret(encoder serde.Encoder, opts *AssetOpts) error

WriteTLSSecret creates a new TLS secret in the kubernetes manifest (equivalent to one generate by 'kubectl create secret tls'). This will be mounted by the pachd pod and used as its TLS public certificate and private key

Types

type AmazonCreds added in v1.7.1 

type AmazonCreds struct {
	// Direct credentials. Only applicable if Pachyderm is given its own permanent
	// AWS credentials
	ID     string // Access Key ID
	Secret string // Secret Access Key
	Token  string // Access token (if using temporary security credentials

	// Vault options (if getting AWS credentials from Vault)
	VaultAddress string // normally addresses come from env, but don't have vault service name
	VaultRole    string
	VaultToken   string
}

AmazonCreds are options that are applicable specifically to Pachd's credentials in an AWS deployment

type AssetOpts added in v1.2.4 

type AssetOpts struct {
	FeatureFlags
	StorageOpts
	PachdShards uint64
	Version     string
	LogLevel    string
	Metrics     bool
	Dynamic     bool
	EtcdNodes   int
	EtcdVolume  string
	DashOnly    bool
	NoDash      bool
	DashImage   string
	Registry    string
	EtcdPrefix  string
	PachdPort   int32
	TracePort   int32
	HTTPPort    int32
	PeerPort    int32

	// NoGuaranteed will not generate assets that have both resource limits and
	// resource requests set which causes kubernetes to give the pods
	// guaranteed QoS. Guaranteed QoS generally leads to more stable clusters
	// but on smaller test clusters such as those run on minikube it doesn't
	// help much and may cause more instability than it prevents.
	NoGuaranteed bool

	// DisableAuthentication stops Pachyderm's authentication service
	// from talking to GitHub, for testing. Instead users can authenticate
	// simply by providing a username.
	DisableAuthentication bool

	// BlockCacheSize is the amount of memory each PachD node allocates towards
	// its cache of PFS blocks. If empty, assets.go will choose a default size.
	BlockCacheSize string

	// PachdCPURequest is the amount of CPU we request for each pachd node. If
	// empty, assets.go will choose a default size.
	PachdCPURequest string

	// PachdNonCacheMemRequest is the amount of memory we request for each
	// pachd node in addition to BlockCacheSize. If empty, assets.go will choose
	// a default size.
	PachdNonCacheMemRequest string

	// EtcdCPURequest is the amount of CPU (in cores) we request for each etcd
	// node. If empty, assets.go will choose a default size.
	EtcdCPURequest string

	// EtcdMemRequest is the amount of memory we request for each etcd node. If
	// empty, assets.go will choose a default size.
	EtcdMemRequest string

	// EtcdStorageClassName is the name of an existing StorageClass to use when
	// creating a StatefulSet for dynamic etcd storage. If unset, a new
	// StorageClass will be created for the StatefulSet.
	EtcdStorageClassName string

	// IAM role that the Pachyderm deployment should assume when talking to AWS
	// services (if using kube2iam + metadata service + IAM role to delegate
	// permissions to pachd via its instance).
	// This is in AssetOpts rather than AmazonCreds because it must be passed
	// as an annotation on the pachd pod rather than as a k8s secret
	IAMRole string

	// ImagePullSecret specifies an image pull secret that gets attached to the
	// various deployments so that their images can be pulled from a private
	// registry.
	ImagePullSecret string

	// NoRBAC, if true, will disable creation of RBAC assets.
	NoRBAC bool

	// LocalRoles, if true, uses Role and RoleBinding instead of ClusterRole and
	// ClusterRoleBinding.
	LocalRoles bool

	// Namespace is the kubernetes namespace to deploy to.
	Namespace string

	// NoExposeDockerSocket if true prevents pipelines from accessing the docker socket.
	NoExposeDockerSocket bool

	// ExposeObjectAPI, if set, causes pachd to serve Object/Block API requests on
	// its public port. This should generally be false in production (it breaks
	// auth) but is needed by tests
	ExposeObjectAPI bool

	// If set, the files indictated by 'TLS.ServerCert' and 'TLS.ServerKey' are
	// placed into a Kubernetes secret and used by pachd nodes to authenticate
	// during TLS
	TLS *TLSOpts

	// Sets the cluster deployment ID. If unset, this will be a randomly
	// generated UUID without dashes.
	ClusterDeploymentID string

	// RequireCriticalServersOnly is true when only the critical Pachd servers
	// are required to startup and run without error.
	RequireCriticalServersOnly bool
}

AssetOpts are options that are applicable to all the asset types.

type FeatureFlags added in v1.8.6 

type FeatureFlags struct {
	// NewStorageLayer, if true, will make Pachyderm use the new storage layer.
	NewStorageLayer bool
}

FeatureFlags are flags for experimental features.

type StorageOpts added in v1.9.11 

type StorageOpts struct {
	UploadConcurrencyLimit int
}

StorageOpts are options that are applicable to the storage layer.

type TLSOpts added in v1.7.4 

type TLSOpts struct {
	ServerCert string
	ServerKey  string
}

TLSOpts indicates the cert and key file that Pachd should use to authenticate with clients

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.