backup

package
v0.0.0-...-4b711b1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 27, 2024 License: Apache-2.0 Imports: 65 Imported by: 3

Documentation

Index

Constants

View Source 
const MinioDeleteBatchSize = 512
View Source 
const SnapshotIntegrityMetadataFileName = "snapshot.integrity"

Variables

View Source 
var ErrSnapshotExists error = errors.New("Snapshot already exists")

Functions

func ConsumeStream 

func ConsumeStream(stream ArtifactStream) error

ConsumeStream reads all elements in the stream until EOF is reached.

func IsNotExist 

func IsNotExist(err error) bool

IsNotExist returns true if the error represents a object access against a objects that does not exist

func IsSnapshotChecksumError 

func IsSnapshotChecksumError(err error) bool

func LoadAllSpecsFromBackup 

func LoadAllSpecsFromBackup(ctx context.Context, bucket Bucket, verifier ObjectVerifier) ([]Spec, []Specless, error)

func LoadDeploymentConfig 

func LoadDeploymentConfig(ctx context.Context, bucket Bucket, verifier ObjectVerifier) (*deployment.AutomateConfig, error)

LoadDeploymentConfig loads the automate configuration directly from the bolt database contained in the given backup.

func SetCommandExecutor 

func SetCommandExecutor(spec Spec, exec command.Executor)

SetCommandExecutor sets the command executor for a given spec

func ShowBackupChecksum 

func ShowBackupChecksum(bucket Bucket) (string, error)

func ToObjectPaths 

func ToObjectPaths(deleteObjs []BucketObject) []string

Types

type ArtifactRepo 

type ArtifactRepo struct {
	// contains filtered or unexported fields
}

func NewArtifactRepo 

func NewArtifactRepo(backupLocationSpec LocationSpecification) *ArtifactRepo

func (*ArtifactRepo) ListArtifacts 

func (repo *ArtifactRepo) ListArtifacts(ctx context.Context) ArtifactStream

ListArtifacts returns a stream of the actual object names from the bucket.

func (*ArtifactRepo) ListArtifactsFromSnapshots 

func (repo *ArtifactRepo) ListArtifactsFromSnapshots(ctx context.Context, excludedSnapshots ...string) ArtifactStream

ListArtifactsFromSnapshots aggregates all the required artifacts from all snapshots and returns a stream. It is presumed to be much cheaper to scan every snapshot metadata file and aggregate the artifacts rather than list every object that is currently in the repo. The downside of course is that we aren't sure that such objects exist.

func (*ArtifactRepo) ReadSnapshotIntegrityMetadata 

func (repo *ArtifactRepo) ReadSnapshotIntegrityMetadata(ctx context.Context) (*ArtifactRepoIntegrityMetadata, error)

ReadSnapshotIntegrityMetadata reads the snapshot integrity metadata file

func (*ArtifactRepo) Remove 

func (repo *ArtifactRepo) Remove(ctx context.Context, name string) error

func (*ArtifactRepo) Restore 

func (repo *ArtifactRepo) Restore(ctx context.Context, dstBucket Bucket, name string, optFuncs ...ArtifactRestoreOpt) error

func (*ArtifactRepo) Snapshot 

func (repo *ArtifactRepo) Snapshot(ctx context.Context, name string, srcBucket Bucket,
	requiredArtifacts ArtifactStream, optFuncs ...ArtifactSnapshotOpt) (ArtifactRepoSnapshotMetadata, error)

func (*ArtifactRepo) ValidateSnapshotIntegrity 

func (repo *ArtifactRepo) ValidateSnapshotIntegrity(ctx context.Context, filters ...FilterSnapshotOpt) error

ValidateSnapshotIntegrity validates snapshot integrity by scanning the repo artifacts list and ensuring that each snapshot's required artifacts are present.

type ArtifactRepoIntegrityMetadata 

type ArtifactRepoIntegrityMetadata struct {
	Snapshots map[string]ArtifactRepoSnapshotIntegrityMetadata `json:"snapshots"`
}

func NewArtifactRepoIntegrityMetadata 

func NewArtifactRepoIntegrityMetadata() *ArtifactRepoIntegrityMetadata

type ArtifactRepoProgressReporter 

type ArtifactRepoProgressReporter interface {
	ReportProgress(completed int64, total int64)
}

type ArtifactRepoSnapshotIntegrityMetadata 

type ArtifactRepoSnapshotIntegrityMetadata struct {
	LastVerified string   `json:"last_verified"`
	Missing      []string `json:"missing"`
	Corrupted    bool     `json:"corrupted"`
}

func NewArtifactRepoSnapshotIntegrityMetadata 

func NewArtifactRepoSnapshotIntegrityMetadata() ArtifactRepoSnapshotIntegrityMetadata

type ArtifactRepoSnapshotMetadata 

type ArtifactRepoSnapshotMetadata struct {
	Name     string `json:"name"`
	Checksum string `json:"checksum"`
}

type ArtifactRestoreOpt 

type ArtifactRestoreOpt func(*artifactRestoreOptions)

func ArtifactRepoRestoreReportProgress 

func ArtifactRepoRestoreReportProgress(progress ArtifactRepoProgressReporter) ArtifactRestoreOpt

func ArtifactRepoRestoreValidateChecksum 

func ArtifactRepoRestoreValidateChecksum(checksum string) ArtifactRestoreOpt

type ArtifactSnapshotOpt 

type ArtifactSnapshotOpt func(*artifactSnapshotOptions)

func ArtifactRepoSnapshotReportProgress 

func ArtifactRepoSnapshotReportProgress(progress ArtifactRepoProgressReporter) ArtifactSnapshotOpt

type ArtifactStream 

type ArtifactStream interface {
	Next() (string, error)
	Close() error
}

ArtifactStream is an iterator of artifacts keys by name. A stream may not produce the same value twice. The values produced by a stream are sorted. It is generally not safe to reuse a stream or use them concurrently. Returns io.EOF when no more elements will be produced

func EmptyStream 

func EmptyStream() ArtifactStream

EmptyStream is an ArtifactStream that always returns io.EOF

func ErrStream 

func ErrStream(err error) ArtifactStream

ErrStream is an ArtifactStream that always returns the given error

func Merge 

func Merge(streams ...ArtifactStream) ArtifactStream

Merge returns a stream which is the union of all the given streams. Values returned will be unique and sorted.

func NewArrayStream 

func NewArrayStream(items []string) ArtifactStream

func NewBucketObjectNameStream 

func NewBucketObjectNameStream(ctx context.Context, bucket Bucket, pathPrefix string) (ArtifactStream, error)

NewBucketObjectNameStream takes a bucket and returns a streams of all object names for the given path

func NewLineReaderStream 

func NewLineReaderStream(reader io.ReadCloser) ArtifactStream

NewLineReaderStream returns an artifact stream that reads line by line from the provided reader, and produces each line as an item

func NewLoggingStream 

func NewLoggingStream(stream ArtifactStream, out io.Writer) ArtifactStream

NewLoggingStream returns a stream which writes the produced items to the provided writer as the stream is read.

func Sub 

func Sub(a ArtifactStream, b ArtifactStream) ArtifactStream

Sub returns a stream with elements in a but not in b a - b

func Xor 

func Xor(a ArtifactStream, b ArtifactStream) ArtifactStream

Xor returns a stream with elements in a or b but not both a xor b

type BackupStatus 

type BackupStatus string
const (
	// BackupStatusCompleted is written to a backup that has been
	// completed successfully
	BackupStatusCompleted BackupStatus = "completed"
	// BackupStatusInProgress is written to a backup when it starts
	BackupStatusInProgress BackupStatus = "in_progress"
	// BackupStatusFailed is written to a backup when it fails
	BackupStatusFailed BackupStatus = "failed"
	// BackupStatusDeleting is written to a backup we start deleting it
	BackupStatusDeleting BackupStatus = "deleting"
)

type BlobReader 

type BlobReader interface {
	io.ReadCloser
	BlobSHA256() string
}

type BlobUploadRequest 

type BlobUploadRequest struct {
	// Key is the key corresponding to the data that will be returned by Reader
	Key string
	// Reader is a reader for that data for Key
	Reader io.ReadCloser
}

type BlobWriter 

type BlobWriter interface {
	io.WriteCloser
	Fail(error) error
	BlobSHA256() string
}

A BlobWriter is a WriteCloser that can also be failed

type Bucket 

type Bucket interface {
	// NewReader returns a ReadCloser for the given storage key. The reader must be closed
	// by the caller. It is provided only when there is not an error.
	// If a object with the given name is not found, calling IsNotExist with the error
	// must return true
	NewReader(ctx context.Context, path string, verifier ObjectVerifier) (io.ReadCloser, error)

	// NewWriter returns a BlobWriter that can write to the given storage key. The data may
	// not be flushed until Close is called
	NewWriter(ctx context.Context, path string) (BlobWriter, error)

	// List returns a list of BucketObjects with the given object name prefix.
	// If there are no matching objects stored, empty lists will be returned.
	//
	// If delimited is set to true, SharedPrefix objects will be returned. Doing this is
	// equivalent to asking for a 1 level deep directory search. As an example, if pathPrefix
	// was set to 'foo/', then the directory 'foo/bar' is a possible return value, but
	// not 'foo/bar/baz'. If 'foo/quux' exists and is an object, then it would be returned in the
	// BucketObject list.
	//
	// The pathPrefix provided is assumed to end in '/'
	List(ctx context.Context, pathPrefix string, delimited bool) ([]BucketObject, []SharedPrefix, error)

	Delete(ctx context.Context, objectPaths []string) error
}

A Bucket is a place where blobs can be stored

func NewFilesystemBucket 

func NewFilesystemBucket(basePath string) Bucket

NewFilesystemBucket returns a Bucket backed by a filesystem directory

func NewGCSBucket 

func NewGCSBucket(name string, basePath string, c *GCSConfig) (Bucket, error)

func NewS3Bucket 

func NewS3Bucket(name string, basePath string, c *aws.Config) (Bucket, error)

type BucketObject 

type BucketObject struct {
	// The name is the name of the object in the bucket relative to the bucket
	// path. It can be a singular object name or a relative path.
	// For example, if we instantiated an s3 bucket with the bucket name
	// of foo, and a base path of bar/baz, all access to object would
	// be implicitly relative to s3://foo/bar/baz. Calling
	// NewReader(ctx, "quuz") would access s3://foo/bar/baz/quux
	Name string
}

BucketObject represents the metadata for a stored object

type BuilderMinioDumpOperation 

type BuilderMinioDumpOperation struct {
	Name       string   `json:"name"`
	ObjectName []string `json:"storage_key"`
}

func (*BuilderMinioDumpOperation) Backup 

func (d *BuilderMinioDumpOperation) Backup(backupCtx Context, om ObjectManifest, progChan chan OperationProgress) error

func (*BuilderMinioDumpOperation) Delete 

func (d *BuilderMinioDumpOperation) Delete(backupCtx Context) error

func (*BuilderMinioDumpOperation) Restore 

func (d *BuilderMinioDumpOperation) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier, progChan chan OperationProgress) error

func (*BuilderMinioDumpOperation) String 

func (d *BuilderMinioDumpOperation) String() string

type BuilderMinioDumpOperationV2 

type BuilderMinioDumpOperationV2 struct {
	Name       string   `json:"name"`
	ObjectName []string `json:"storage_key"`
}

func (*BuilderMinioDumpOperationV2) Backup 

func (d *BuilderMinioDumpOperationV2) Backup(backupCtx Context, om ObjectManifest, progChan chan OperationProgress) error

func (*BuilderMinioDumpOperationV2) Delete 

func (d *BuilderMinioDumpOperationV2) Delete(backupCtx Context) error

func (*BuilderMinioDumpOperationV2) Restore 

func (d *BuilderMinioDumpOperationV2) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier,
	progChan chan OperationProgress) error

func (*BuilderMinioDumpOperationV2) String 

func (d *BuilderMinioDumpOperationV2) String() string

type BulkDeleteIterator 

type BulkDeleteIterator interface {
	// Next will return objects that need to be delete
	Next() (string, error)
}

type BulkDeleter 

type BulkDeleter interface {
	Delete(context.Context, BulkDeleteIterator) error
}

func NewBulkDeleter 

func NewBulkDeleter(bkt Bucket, prefix string) BulkDeleter

type BulkUploadIterator 

type BulkUploadIterator interface {
	// Next will return objects that need to be uploaded
	Next() (BlobUploadRequest, error)
	EstimatedSize() int64
}

type BulkUploader 

type BulkUploader interface {
	Upload(context.Context, BulkUploadIterator, ArtifactRepoProgressReporter) error
}

func NewBulkUploader 

func NewBulkUploader(dst Bucket, prefix string) BulkUploader

type CancellableTask 

type CancellableTask struct {
	Status *api.BackupStatusResponse
	Cancel func()
}

type Cmd 

type Cmd struct {
	Name    string   `json:"name"`
	Dump    []string `json:"dump"`
	Restore []string `json:"restore"`
}

Cmd represents a command to be run to dump/restore data from an a2 service. Commands provide a way to backup/restore data that's not stored in postgres or elasticsearch and cannot be safely copied from a service's data dir.

type CommandExecuteOperation 

type CommandExecuteOperation struct {
	Name       string   `json:"name"`
	ObjectName []string `json:"storage_key"`
	Cmd        Cmd      `json:"cmd"`
	PkgOrigin  string   `json:"pkg_origin"`
	PkgName    string   `json:"pkg_name"`
	// contains filtered or unexported fields
}

CommandExecuteOperation represets a command running execution

func (*CommandExecuteOperation) Backup 

func (c *CommandExecuteOperation) Backup(backupCtx Context, om ObjectManifest, progChan chan OperationProgress) error

Backup executes a backup specification.

func (*CommandExecuteOperation) Delete 

func (c *CommandExecuteOperation) Delete(backupCtx Context) error

func (*CommandExecuteOperation) Restore 

func (c *CommandExecuteOperation) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier, progChan chan OperationProgress) error

Restore executes a backup specification.

func (*CommandExecuteOperation) String 

func (c *CommandExecuteOperation) String() string

String returns the string representation of the operation

type Context 

type Context struct {
	// contains filtered or unexported fields
}

Context is stuff

func NewContext 

func NewContext(opts ...ContextOpt) Context

NewContext returns a new backup context

func (Context) DeleteBackupMetadata 

func (ctx Context) DeleteBackupMetadata() error

func (*Context) IsExternalPG 

func (ctx *Context) IsExternalPG() bool

func (Context) MetadataWritten 

func (ctx Context) MetadataWritten(mdName string, mdData []byte)

func (Context) Transact 

func (ctx Context) Transact(f TransactFunc) (string, error)

Transact runs f. It will first write a status object with 'pending'. If the function runs without error, the status object will be overwritten with 'completed'. Otherwise, it will be overwritten with 'failed'.

NOTE: Transact used to move a completed backup into place atomically. This 

assumed that we could do a rename. We now use a blob like api, and
there exists no atomic rename. With the blob api, this changed to
using a status file. Since the status file did not exist before,
a successful backup is one where either there is no status file,
or one where the status file contains 'completed'.

func (Context) TransactDelete 

func (ctx Context) TransactDelete(f TransactFunc) error

type ContextOpt 

type ContextOpt func(*Context)

ContextOpt represents an optional configuration function for a Runner

func WithContextBackupLocationSpecification 

func WithContextBackupLocationSpecification(locationSpec LocationSpecification) ContextOpt

WithContextBackupLocationSpecification configures the backup-gateway location

func WithContextBackupRestoreLocationSpecification 

func WithContextBackupRestoreLocationSpecification(locationSpec LocationSpecification) ContextOpt

WithContextRestoreLocationSpecification configures the backup restore remote location

func WithContextBackupRestoreTask 

func WithContextBackupRestoreTask(task *api.BackupRestoreTask) ContextOpt

WithContextBackupRestoreTask configures the context backup task

func WithContextBackupTask 

func WithContextBackupTask(task *api.BackupTask) ContextOpt

WithContextBackupTask configures the context backup task

func WithContextBuilderMinioLocationSpec 

func WithContextBuilderMinioLocationSpec(locationSpec LocationSpecification) ContextOpt

WithContextBuilderMinioLocationSpec configures the builder minio remote location

func WithContextConnFactory 

func WithContextConnFactory(connFactory *secureconn.Factory) ContextOpt

WithContextConnFactory configures the secureconn.Factory to use to connect to services

func WithContextCtx 

func WithContextCtx(ctx2 context.Context) ContextOpt

WithContextCtx :shrug: configures the context's context...

func WithContextEsSidecarInfo 

func WithContextEsSidecarInfo(esSidecarInfo ESSidecarConnInfo) ContextOpt

WithContextEsSidecarInfo configures the EsSidecar connection info

func WithContextExternalPG 

func WithContextExternalPG(isExternal bool) ContextOpt

WithContextExternalPG sets the postgres mode. If external pg is enabled, certain things like role creation will not happen.

func WithContextPgConnInfo 

func WithContextPgConnInfo(info pg.ConnInfo) ContextOpt

WithContextPgConnInfo configures the context postgres connection info

func WithContextReleaseManifest 

func WithContextReleaseManifest(releaseManifest manifest.ReleaseManifest) ContextOpt

WithContextReleaseManifest sets the release manifest. We need this in our testing environment to pick up the right version of things

type CountingStream 

type CountingStream interface {
	ArtifactStream
	Count() int64
}

CountingStream is a stream that keeps track of the number of items procured

func NewCountingStream 

func NewCountingStream(stream ArtifactStream) CountingStream

NewCountingStream returns a stream which keeps track of the number of items produced

type DatabaseDumpOperation 

type DatabaseDumpOperation struct {
	Name       string   `json:"name"`
	User       string   `json:"user"`
	ObjectName []string `json:"storage_key"`
	// contains filtered or unexported fields
}

DatabaseDumpOperation represents a database that the service depends on

func (*DatabaseDumpOperation) Backup 

func (d *DatabaseDumpOperation) Backup(backupCtx Context, om ObjectManifest, progChan chan OperationProgress) error

Backup executes a backup operation.

func (*DatabaseDumpOperation) Delete 

func (d *DatabaseDumpOperation) Delete(backupCtx Context) error

func (*DatabaseDumpOperation) Restore 

func (d *DatabaseDumpOperation) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier, progChan chan OperationProgress) error

Restore executes a backup operation.

func (*DatabaseDumpOperation) String 

func (d *DatabaseDumpOperation) String() string

String returns the string representation of the operation

type DatabaseDumpOperationV2 

type DatabaseDumpOperationV2 struct {
	Name       string   `json:"name"`
	User       string   `json:"user"`
	ObjectName []string `json:"storage_key"`
	// contains filtered or unexported fields
}

DatabaseDumpOperationV2 represents a database that the service depends on.

func (*DatabaseDumpOperationV2) Backup 

func (d *DatabaseDumpOperationV2) Backup(backupCtx Context, om ObjectManifest, progChan chan OperationProgress) error

Backup executes a backup operation.

func (*DatabaseDumpOperationV2) Delete 

func (d *DatabaseDumpOperationV2) Delete(backupCtx Context) error

func (*DatabaseDumpOperationV2) Restore 

func (d *DatabaseDumpOperationV2) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier, progChan chan OperationProgress) error

Restore executes a backup operation.

func (*DatabaseDumpOperationV2) String 

func (d *DatabaseDumpOperationV2) String() string

String returns the string representation of the operation

type ESSidecarConnInfo 

type ESSidecarConnInfo struct {
	Host string
	Port int32
}

ESSidecarConnInfo represents how to connect to the sidecar

func (ESSidecarConnInfo) Address 

func (e ESSidecarConnInfo) Address() string

Address returns an address that can be passed to grpc

type ElasticsearchOperation 

type ElasticsearchOperation struct {
	ServiceName    string
	MultiIndexSpec string
}

ElasticsearchOperation represents a index in ES a service needs backed up / restored

func (*ElasticsearchOperation) Backup 

func (esop *ElasticsearchOperation) Backup(backupCtx Context, _ ObjectManifest, progChan chan OperationProgress) error

Backup backs up indices in ES

func (*ElasticsearchOperation) Delete 

func (esop *ElasticsearchOperation) Delete(backupCtx Context) error

func (*ElasticsearchOperation) Restore 

func (esop *ElasticsearchOperation) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier, progChan chan OperationProgress) error

Restore executes a backup operation.

func (*ElasticsearchOperation) String 

func (esop *ElasticsearchOperation) String() string

type EventHandler 

type EventHandler struct {
	// contains filtered or unexported fields
}

EventHandler is a client side backup event stream handler.

func NewEventHandler 

func NewEventHandler(opts ...EventHandlerOpt) *EventHandler

NewEventHandler returns a new instance of a backup event handler

func (*EventHandler) Clear 

func (eh *EventHandler) Clear()

Clear clears the screen

func (*EventHandler) HandleEventError 

func (eh *EventHandler) HandleEventError(event *api.DeployEvent) (completed bool, err error)

HandleEventError takes a deployment event, parses for backup event types and handles them. If an event is a completion event it'll return that the stream has completed.

type EventHandlerOpt 

type EventHandlerOpt func(*EventHandler)

EventHandlerOpt represents an configuration function for the event handler

func WithWriter 

func WithWriter(writer cli.FormatWriter) EventHandlerOpt

WithWriter configures the event sender's writer

type Executor 

type Executor struct {
	// contains filtered or unexported fields
}

Executor executes backup operations. It listens for the operation events, annotates them, and publishes them to the status and publishing the backup status to event sender.

func NewExecutor 

func NewExecutor(opts ...ExecutorOpt) *Executor

NewExecutor returns a new instance of a backup job executor

func (*Executor) Backup 

func (b *Executor) Backup(backupCtx Context) error

Backup runs the operations and waits for them to complete. When completed or failed it publishes notifications to the event channel and notifies that runner that the operations have completed.

func (*Executor) DeleteBackup 

func (b *Executor) DeleteBackup(backupCtx Context) error

func (*Executor) Restore 

func (b *Executor) Restore(backupCtx Context, metadata *Metadata) error

Restore does a restores of all backup operations in the spec

func (*Executor) RestoreSyncCmds 

func (b *Executor) RestoreSyncCmds(backupCtx Context, metadata *Metadata) error

RestoreSyncCmds synchronously restores cmd data of the spec

func (*Executor) RestoreSyncPaths 

func (b *Executor) RestoreSyncPaths(backupCtx Context, metadata *Metadata) error

RestoreSyncPaths synchronously restores path data of the spec

type ExecutorOpt 

type ExecutorOpt func(*Executor)

ExecutorOpt represents an optional configuration function for an Executor

func WithCancel 

func WithCancel(canx func()) ExecutorOpt

WithCancel configures the backup execution context cancellation callback

func WithErrorChan 

func WithErrorChan(c chan error) ExecutorOpt

WithErrorChan configures the error channel that the executor will publish if an operation fails.

func WithEventChan 

func WithEventChan(c chan api.DeployEvent_Backup_Operation) ExecutorOpt

WithEventChan configures the event sender channel that operation events and published to.

func WithLock 

func WithLock(lock *sync.Mutex) ExecutorOpt

WithLock sets the lock an executor must hold before it may do any actual backup task

func WithSpec 

func WithSpec(s Spec) ExecutorOpt

WithSpec configures the backup operations from a given Spec

type FilesystemLocationSpecification 

type FilesystemLocationSpecification struct {
	Path string
}

func (FilesystemLocationSpecification) ConfigureBackupRestoreTask 

func (fsspec FilesystemLocationSpecification) ConfigureBackupRestoreTask(req *api.BackupRestoreTask) error

func (FilesystemLocationSpecification) String 

func (fsspec FilesystemLocationSpecification) String() string

func (FilesystemLocationSpecification) ToBucket 

func (fsspec FilesystemLocationSpecification) ToBucket(baseKey string) Bucket

type FilterSnapshotOpt 

type FilterSnapshotOpt func(*filterSnapshotOptions)

func ExcludeSnapshots 

func ExcludeSnapshots(snapshots []string) FilterSnapshotOpt

func OnlySnapshots 

func OnlySnapshots(snapshots []string) FilterSnapshotOpt

type GCSConfig 

type GCSConfig struct {
	GoogleApplicationCredentials string
}

type GCSLocationSpecification 

type GCSLocationSpecification struct {
	// Required
	BucketName string

	// Optional
	BasePath                     string
	GoogleApplicationCredentials string
}

func (GCSLocationSpecification) ConfigureBackupRestoreTask 

func (gcsSpec GCSLocationSpecification) ConfigureBackupRestoreTask(req *api.BackupRestoreTask) error

func (GCSLocationSpecification) String 

func (gcsSpec GCSLocationSpecification) String() string

func (GCSLocationSpecification) ToBucket 

func (gcsSpec GCSLocationSpecification) ToBucket(baseKey string) Bucket

type GatewayLocationSpecification 

type GatewayLocationSpecification struct {
	BucketName string
	Endpoint   string
	BasePath   string
	CertPool   *x509.CertPool
	// contains filtered or unexported fields
}

GatewayLocationSpecification describes how to communicate with the backup gateway

func (GatewayLocationSpecification) ConfigureBackupRestoreTask 

func (gws GatewayLocationSpecification) ConfigureBackupRestoreTask(req *api.BackupRestoreTask) error

ConfigureBackupRestoreTask is a NOOP callback implementation to satisfy the LocationSpecification interface.

func (GatewayLocationSpecification) String 

func (gws GatewayLocationSpecification) String() string

String is the backup gateway identified as a string

func (GatewayLocationSpecification) ToBucket 

func (gws GatewayLocationSpecification) ToBucket(key string) Bucket

ToBucket returns a backup bucket that can be used to communicate with the backup gateway.

type LocationSpecification 

type LocationSpecification interface {
	// ToBucket converts the location specification to a bucket. All accessess have an
	// implicit prefix of baseKey
	ToBucket(baseKey string) Bucket

	// ConfigureBackupRestoreTask configures the task so that its backup location is
	// consistent with self
	ConfigureBackupRestoreTask(*api.BackupRestoreTask) error

	// String returns a string representation
	String() string
}

LocationSpecification describes where data lives. It returns a bucket to write to

func NewBackupGatewayLocationSpec 

func NewBackupGatewayLocationSpec(endpoint,
	bucketName,
	basePath string,
	rootCert []byte,
	secretStore secrets.SecretsReader) (LocationSpecification, error)

NewBackupGatewayLocationSpec takes the backup gateway endpoint, bucket name, base path, root TLS certificate and a secret store and returns a compatible backup location specification.

func NewMinioLocationSpec 

func NewMinioLocationSpec(endpoint,
	bucketName,
	basePath string,
	groupName string,
	rootCert []byte,
	secretStore secrets.SecretsReader) (LocationSpecification, error)

func NewRemoteLocationSpecificationFromGlobalConfig 

func NewRemoteLocationSpecificationFromGlobalConfig(globalConfig *config.GlobalConfig) LocationSpecification

NewRemoteLocationSpecificationFromGlobalConfig takes the GlobalConfig and converts it into a corresponding LocationSpecification type depending on the backup location.

func NewRemoteLocationSpecificationFromRestoreTask 

func NewRemoteLocationSpecificationFromRestoreTask(restoreTask *api.BackupRestoreTask) LocationSpecification

NewRemoteLocationSpecificationFromRestoreTask takes BackupRestoreTask and converts it into a corresponding LocationSpecification type depending on the backup location.

type Metadata 

type Metadata struct {
	Spec                     *Spec             `json:"spec"`
	Task                     *api.BackupTask   `json:"task"`
	DeploymentServiceVersion string            `json:"deployment_service_version"`
	ContentsSHA256           map[string]string `json:"contents_sha256"`
}

Metadata is a wrapper struct for metadata that should be backed up.

func LoadServiceMetadata 

func LoadServiceMetadata(ctx context.Context, bucket Bucket, svcName string, objectVerifier ObjectVerifier) (*Metadata, error)

LoadServiceMetadata takes a backup bucket and service name and returns a new Metadata instance from the backed up metadata in the given bucket. Errors caused by the metadata not existing will return true when passed to `backup.IsNotExist(err)`

func (*Metadata) Verifier 

func (m *Metadata) Verifier() ObjectVerifier

type MetadataChecksums 

type MetadataChecksums struct {
	ContentsSHA256 map[string]string `json:"contents_sha256"`
}

type MetadataWriterOperation 

type MetadataWriterOperation struct {
	Spec       *Spec    `json:"spec"`
	ObjectName []string `json:"storage_key"`
}

MetadataWriterOperation represents an operation that writes the metadata.json

func (*MetadataWriterOperation) Backup 

func (m *MetadataWriterOperation) Backup(backupCtx Context, om ObjectManifest, progChan chan OperationProgress) error

Backup executes a backup specification.

func (*MetadataWriterOperation) Delete 

func (m *MetadataWriterOperation) Delete(backupCtx Context) error

func (*MetadataWriterOperation) Restore 

func (m *MetadataWriterOperation) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier, progChan chan OperationProgress) error

Restore executes a backup specification.

func (*MetadataWriterOperation) String 

func (m *MetadataWriterOperation) String() string

String returns the string representation of the operation

type NoOpObjectVerifier 

type NoOpObjectVerifier struct {
}

NoOpObjectVerifier implements the ObjectVerifier interface but returns the success case for all methods. It's here for compatibility with older backups that don't have any checksums.

func (*NoOpObjectVerifier) ObjectValid 

func (o *NoOpObjectVerifier) ObjectValid(objectName string) error

func (*NoOpObjectVerifier) ValidateBlobContent 

func (o *NoOpObjectVerifier) ValidateBlobContent(objectName string, r BlobReader) error

type ObjectManifest 

type ObjectManifest struct {
	// contains filtered or unexported fields
}

func NewObjectManifest 

func NewObjectManifest() ObjectManifest

func (ObjectManifest) DataWritten 

func (o ObjectManifest) DataWritten(objectName string, data []byte)

func (ObjectManifest) ObjectSHA256s 

func (o ObjectManifest) ObjectSHA256s() map[string]string

func (ObjectManifest) WriteFinished 

func (o ObjectManifest) WriteFinished(objectName string, w BlobWriter)

type ObjectVerifier 

type ObjectVerifier interface {
	// ObjectValid checks if the object is present in the backup metadata to
	// prevent a malicious actor from sneaking an extra file into a backup.
	ObjectValid(objectName string) error
	ValidateBlobContent(objectName string, r BlobReader) error
}

ObjectVerifier checks objects as they are restored to ensure they have the correct checksums as recorded when they were backed up

func LoadMetadataVerifier 

func LoadMetadataVerifier(ctx context.Context, bucket Bucket, sha256 string) (ObjectVerifier, error)

LoadMetadataVerifier returns an ObjectVerifier that can be used to verify the integrity of the metadata.json file for each component backup. The metadata.json files then contain checksums of the individual files in the per-service backup.

type Operation 

type Operation interface {
	Backup(ctx Context, om ObjectManifest, prog chan OperationProgress) error
	Restore(ctx Context, serviceName string, verifier ObjectVerifier, prog chan OperationProgress) error
	Delete(ctx Context) error
	String() string
}

Operation is an interface for a backup operation.

type OperationProgress 

type OperationProgress struct {
	Name     string
	Progress float64
}

OperationProgress represents an operations current progress

type PathCopyOperation 

type PathCopyOperation struct {
	Name          string         `json:"name"`
	ObjectName    []string       `json:"storage_key"`
	SrcPath       string         `json:"src_path"`
	RsyncMatchers []RsyncMatcher `json:"-"` // don't marshal an interface
	Owner         string
	// contains filtered or unexported fields
}

PathCopyOperation backups a source path to a destination path. It also supports a slice of optional Rsync matchers to include or exclude files or directories from the source path.

func (*PathCopyOperation) Backup 

func (p *PathCopyOperation) Backup(backupCtx Context, om ObjectManifest, progChan chan OperationProgress) error

Backup backs up a path using the defined fields on the struct and publishes progress events to the progress channel.

func (*PathCopyOperation) Delete 

func (p *PathCopyOperation) Delete(backupCtx Context) error

func (*PathCopyOperation) Restore 

func (p *PathCopyOperation) Restore(backupCtx Context, serviceName string, verifier ObjectVerifier, progChan chan OperationProgress) error

Restore restores a path using the defined fields on the struct and publishes progress events to the progress channel.

func (*PathCopyOperation) String 

func (p *PathCopyOperation) String() string

String returns the string representation of the operation

type PeekableArtifactStream 

type PeekableArtifactStream interface {
	ArtifactStream
	// Peek returns the next element that will be returned
	// when Next is called. It can be called multiple times
	// and will keep generating the same value until Next
	// is called
	Peek() (string, error)
}

PeekableArtifactStream is an artifact stream that allows peeking at the next element without consuming it

func NewPeekableStream 

func NewPeekableStream(s ArtifactStream) PeekableArtifactStream

NewPeekableStream wraps the given ArtifactStream so that it can be peeked

type ProgressCalculator 

type ProgressCalculator struct {
	// contains filtered or unexported fields
}

ProgressCalculator tracks the progress of multiple operations

func NewProgressCalculator 

func NewProgressCalculator() *ProgressCalculator

NewProgressCalculator returns a new initialized ProgressCalculator

func (*ProgressCalculator) Done 

func (p *ProgressCalculator) Done()

Done sets all operations in the progress calculator to 100%

func (*ProgressCalculator) Percent 

func (p *ProgressCalculator) Percent() float64

Percent returns the average completion percent between all operations in the progress calculator. It's a bit microsoftian in that some operations may run much longer than others.

func (*ProgressCalculator) Update 

func (p *ProgressCalculator) Update(op OperationProgress)

Update updates an operation in the progress calculator

type ReleaseManifestLocation 

type ReleaseManifestLocation interface {
	Provider() manifest.ReleaseManifestProvider
}

func NewBucketManifestLocation 

func NewBucketManifestLocation(ctx context.Context, bucket Bucket) (ReleaseManifestLocation, error)

func NewLatestManifestLocation 

func NewLatestManifestLocation(hartifacts, origin string) ReleaseManifestLocation

func NewOnDiskManifestLocation 

func NewOnDiskManifestLocation(path string) ReleaseManifestLocation

type ReplayableStream 

type ReplayableStream interface {
	ArtifactStream
	Reset() error
}

ReplayableStream is a stream that consumes a stream to that caches a stream and allows it to be replayed over again after the Reset() function is called.

func NewReplayableStream 

func NewReplayableStream(stream ArtifactStream) (ReplayableStream, error)

NewReplayableStream returns a stream which is cached to a temporary file and is replyable if it has been reset.

type RequiredChecksumDataMissingError 

type RequiredChecksumDataMissingError struct {
	// contains filtered or unexported fields
}

func (*RequiredChecksumDataMissingError) Error 

func (e *RequiredChecksumDataMissingError) Error() string

type RsyncExclude 

type RsyncExclude struct {
	Pattern string
}

RsyncExclude is the concreate type for an rsync exclude rule

func (*RsyncExclude) RsyncArgs 

func (r *RsyncExclude) RsyncArgs() (flag, matcher string)

RsyncArgs give the rsync args for the exclude filter

type RsyncInclude 

type RsyncInclude struct {
	Pattern string
}

RsyncInclude is the concrete type for an rsync include rule

func (*RsyncInclude) RsyncArgs 

func (r *RsyncInclude) RsyncArgs() (flag, matcher string)

RsyncArgs gives the rsync args for the include filter

type RsyncMatcher 

type RsyncMatcher interface {
	RsyncArgs() (flag, matcher string)
}

RsyncMatcher is a rsync include or exclude pattern

func Exclude 

func Exclude(rsyncPattern string) RsyncMatcher

Exclude creates an RsyncMatcher representing a `--exclude PATTERN` argument to rsync

func Include 

func Include(rsyncPattern string) RsyncMatcher

Include creates an RsyncMatcher representing a `--include PATTERN` argument to rsync

type Runner 

type Runner struct {

	// map[task-id][service-name] = operation-status
	TaskOperations map[string]map[string]api.DeployEvent_Backup_Operation
	// contains filtered or unexported fields
}

Runner represents a new backup runner

func NewRunner 

func NewRunner(opts ...RunnerOpt) *Runner

NewRunner returns a new instance of a backup job runner

func (*Runner) BackupIntegrityShow 

func (r *Runner) BackupIntegrityShow(ctx context.Context, req *api.BackupIntegrityShowRequest) ([]*api.SnapshotIntegrity, error)

func (*Runner) Cancel 

func (r *Runner) Cancel(ctx context.Context) error

Cancel cancels the currently running backup operation. If the current operation is IDLE it will return an error.

func (*Runner) Configure 

func (r *Runner) Configure(opts ...RunnerOpt) *Runner

Configure updates the instance with RunnerOpt's

func (*Runner) CreateBackup 

func (r *Runner) CreateBackup(ctx context.Context, dep *deployment.Deployment, sender events.EventSender) (*api.BackupTask, error)

CreateBackup creates an Automate Backup

func (*Runner) DeleteBackups 

func (r *Runner) DeleteBackups(ctx context.Context, dep *deployment.Deployment, backupTasks []*api.BackupTask) error

DeleteBackups deletes one or many Automate Backups

func (*Runner) ListBackups 

func (r *Runner) ListBackups(ctx context.Context) ([]*api.BackupTask, error)

ListBackups lists all of the available backups

func (*Runner) RestoreBackup 

func (r *Runner) RestoreBackup(
	ctx context.Context,
	dep *deployment.Deployment,
	sender events.EventSender,
	bgw LocationSpecification,
	remote LocationSpecification,
	rt *api.BackupRestoreTask) (*api.BackupRestoreTask, error)

RestoreBackup starts a backup restoration in a go routine and returns the task.

func (*Runner) RunningTask 

func (r *Runner) RunningTask(ctx context.Context) *CancellableTask

RunningTask returns the currently running backup task

func (*Runner) ShowBackup 

func (r *Runner) ShowBackup(ctx context.Context, t *api.BackupTask) (*api.BackupDescription, error)

func (*Runner) ValidateBackupIntegrity 

func (r *Runner) ValidateBackupIntegrity(ctx context.Context, dep *deployment.Deployment, backupTasks []*api.BackupTask) ([]*api.SnapshotIntegrity, error)

type RunnerOpt 

type RunnerOpt func(*Runner)

RunnerOpt represents an optional configuration function for a Runner

func WithBackupLocationSpecification 

func WithBackupLocationSpecification(locationSpec LocationSpecification) RunnerOpt

WithBackupLocationSpecification sets the backup-gateway location

func WithBuilderMinioLocationSpec 

func WithBuilderMinioLocationSpec(locationSpec LocationSpecification) RunnerOpt

func WithConfigRenderer 

func WithConfigRenderer(f func(*deployment.Service) (string, error)) RunnerOpt

WithConfigRenderer configures the runner to unlock the given deployment.

func WithConnFactory 

func WithConnFactory(connFactory *secureconn.Factory) RunnerOpt

WithConnFactory configures the secureconn.Factory to use to connect to services

func WithDeploymentStore 

func WithDeploymentStore(deploymentStore persistence.DeploymentStore) RunnerOpt

WithDeploymentStore sets the deployment store for the runner

func WithEsSidecarInfo 

func WithEsSidecarInfo(esSidecarInfo ESSidecarConnInfo) RunnerOpt

WithEsSidecarInfo configures the EsSidecar connection info

func WithPGConnInfo 

func WithPGConnInfo(pgConnInfo pg.ConnInfo) RunnerOpt

func WithReleaseManifest 

func WithReleaseManifest(releaseManifest manifest.ReleaseManifest) RunnerOpt

WithReleaseManifest sets the release manifest for the runner

func WithSpecs 

func WithSpecs(specs []Spec) RunnerOpt

WithSpecs configures the backup specifications to run

func WithTarget 

func WithTarget(target target.Target) RunnerOpt

WithTarget sets the runners deployment target

type S3LocationSpecification 

type S3LocationSpecification struct {
	// Required
	BucketName string
	Endpoint   string

	// Optional
	BasePath     string
	AccessKey    string
	SecretKey    string
	SessionToken string
}

func (S3LocationSpecification) ConfigureBackupRestoreTask 

func (s3spec S3LocationSpecification) ConfigureBackupRestoreTask(req *api.BackupRestoreTask) error

func (S3LocationSpecification) String 

func (s3spec S3LocationSpecification) String() string

func (S3LocationSpecification) ToBucket 

func (s3spec S3LocationSpecification) ToBucket(baseKey string) Bucket

type SHA256Verifier 

type SHA256Verifier struct {
	// contains filtered or unexported fields
}

func (*SHA256Verifier) ObjectValid 

func (o *SHA256Verifier) ObjectValid(objectName string) error

func (*SHA256Verifier) ValidateBlobContent 

func (o *SHA256Verifier) ValidateBlobContent(objectName string, r BlobReader) error

type SharedPrefix 

type SharedPrefix string

SharedPrefix represents a group of object names that exist

type SnapshotChecksumMismatchError 

type SnapshotChecksumMismatchError struct {
	// contains filtered or unexported fields
}

func (*SnapshotChecksumMismatchError) Error 

func (s *SnapshotChecksumMismatchError) Error() string

type SnapshotChecksumMissingError 

type SnapshotChecksumMissingError struct {
	// contains filtered or unexported fields
}

func (*SnapshotChecksumMissingError) Error 

func (s *SnapshotChecksumMissingError) Error() string

type Spec 

type Spec struct {
	// Name of the a2 service to be backed up
	Name string `json:"name"`

	// Version of the backup specification. If the specification changes such
	// that a backup created by the older specification cannot be properly
	// restored by the newer specification, then both must be defined and the
	// newer one should be a new version.
	Version int `json:"version"`

	// Write the backup metadata
	WriteMetadata bool `json:"metadata"`

	// Paths to back up. Broken down into which phase of the backup they
	// should run.
	SyncPaths []PathCopyOperation `json:"sync_paths"`

	// Backup commands to run. Broken down into which phase of the backup they
	// should run.
	SyncCmds []CommandExecuteOperation `json:"sync_cmds"`

	// Databases to dump. Broken down into which phase the database dumps should
	// occur. When DB operations have been implemented they should be used here.
	SyncDbsV2 []DatabaseDumpOperationV2 `json:"sync_dbs_v2"`

	SyncEsIndices []ElasticsearchOperation `json:"sync_es"`

	SyncBuilderMinio []BuilderMinioDumpOperation `json:"sync_builder_minio"`

	SyncBuilderMinioV2 []BuilderMinioDumpOperationV2 `json:"sync_builder_minio_v2"`

	// DEPRECATED
	SyncDbs []DatabaseDumpOperation `json:"sync_dbs"`
	// contains filtered or unexported fields
}

Spec describes the operations required to backup a given service

func DefaultSpecs 

func DefaultSpecs(serviceNames []string) []Spec

DefaultSpecs returns a list of backup specifications

func (*Spec) FinalizingOps 

func (s *Spec) FinalizingOps() []Operation

FinalizingOps returns a slice of Operations that should be run after the data backup operations have completed. Currently this includes only the MetadataWriterOperation, if the relevant service supports it.

func (*Spec) SyncOps 

func (s *Spec) SyncOps() []Operation

SyncOps returns a slice of Operations that should be run synchronously

type Specless 

type Specless = string

type TransactFunc 

type TransactFunc func(Context) error

TransactFunc is a function to execute that takes a backup context

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.