Documentation ¶
Index ¶
- func GetProviderConfigForBackupsFromEnv(ctx context.Context) map[string]interface{}
- func GetProviderConfigFromEnv(ctx context.Context, provider string) map[string]interface{}
- type AzureBlobStorageConfig
- type AzureBlobStorageManager
- func (manager *AzureBlobStorageManager) DeleteObjects(ctx context.Context, keys []string) (err error)
- func (manager *AzureBlobStorageManager) Download(ctx context.Context, output *os.File, key string) error
- func (manager *AzureBlobStorageManager) GetConfiguredPrefix() string
- func (manager *AzureBlobStorageManager) GetDownloadKeyFromFileLocation(location string) string
- func (manager *AzureBlobStorageManager) GetObjectNameFromLocation(location string) (string, error)
- func (manager *AzureBlobStorageManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
- func (manager *AzureBlobStorageManager) SetTimeout(timeout time.Duration)
- func (manager *AzureBlobStorageManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
- type DOSpacesConfig
- type DOSpacesManager
- func (manager *DOSpacesManager) DeleteObjects(ctx context.Context, keys []string) error
- func (manager *DOSpacesManager) Download(ctx context.Context, output *os.File, key string) error
- func (manager *DOSpacesManager) GetConfiguredPrefix() string
- func (manager *DOSpacesManager) GetDownloadKeyFromFileLocation(location string) string
- func (manager *DOSpacesManager) GetObjectNameFromLocation(location string) (string, error)
- func (manager *DOSpacesManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
- func (manager *DOSpacesManager) SetTimeout(timeout time.Duration)
- func (manager *DOSpacesManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
- type FileManager
- type FileManagerFactory
- type FileManagerFactoryT
- type FileObject
- type GCSConfig
- type GCSManager
- func (*GCSManager) DeleteObjects(_ context.Context, _ []string) (err error)
- func (manager *GCSManager) Download(ctx context.Context, output *os.File, key string) error
- func (manager *GCSManager) GetConfiguredPrefix() string
- func (manager *GCSManager) GetDownloadKeyFromFileLocation(location string) string
- func (manager *GCSManager) GetObjectNameFromLocation(location string) (string, error)
- func (manager *GCSManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
- func (manager *GCSManager) SetTimeout(timeout time.Duration)
- func (manager *GCSManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
- type MinioConfig
- type MinioManager
- func (manager *MinioManager) DeleteObjects(ctx context.Context, keys []string) (err error)
- func (manager *MinioManager) Download(ctx context.Context, file *os.File, key string) error
- func (manager *MinioManager) GetConfiguredPrefix() string
- func (manager *MinioManager) GetDownloadKeyFromFileLocation(location string) string
- func (manager *MinioManager) GetObjectNameFromLocation(location string) (string, error)
- func (manager *MinioManager) ListFilesWithPrefix(_ context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
- func (manager *MinioManager) ObjectUrl(objectName string) string
- func (manager *MinioManager) SetTimeout(timeout time.Duration)
- func (manager *MinioManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
- type ObjectIterator
- type S3Config
- type S3Manager
- func (manager *S3Manager) DeleteObjects(ctx context.Context, keys []string) (err error)
- func (manager *S3Manager) Download(ctx context.Context, output *os.File, key string) error
- func (manager *S3Manager) GetConfiguredPrefix() string
- func (manager *S3Manager) GetDownloadKeyFromFileLocation(location string) string
- func (manager *S3Manager) GetObjectNameFromLocation(location string) (string, error)
- func (manager *S3Manager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
- func (manager *S3Manager) SetTimeout(timeout time.Duration)
- func (manager *S3Manager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
- type SettingsT
- type UploadOutput
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GetProviderConfigForBackupsFromEnv ¶ added in v1.0.2
GetProviderConfigForBackupsFromEnv returns the provider config
Types ¶
type AzureBlobStorageConfig ¶
type AzureBlobStorageConfig struct { Container string Prefix string AccountName string AccountKey string SASToken string EndPoint *string ForcePathStyle *bool DisableSSL *bool Marker azblob.Marker UseSASTokens bool }
func GetAzureBlogStorageConfig ¶
func GetAzureBlogStorageConfig(config map[string]interface{}) *AzureBlobStorageConfig
type AzureBlobStorageManager ¶
type AzureBlobStorageManager struct { Config *AzureBlobStorageConfig // contains filtered or unexported fields }
func (*AzureBlobStorageManager) DeleteObjects ¶ added in v0.1.10
func (manager *AzureBlobStorageManager) DeleteObjects(ctx context.Context, keys []string) (err error)
func (*AzureBlobStorageManager) GetConfiguredPrefix ¶ added in v0.1.10
func (manager *AzureBlobStorageManager) GetConfiguredPrefix() string
func (*AzureBlobStorageManager) GetDownloadKeyFromFileLocation ¶ added in v0.1.10
func (manager *AzureBlobStorageManager) GetDownloadKeyFromFileLocation(location string) string
func (*AzureBlobStorageManager) GetObjectNameFromLocation ¶ added in v0.1.10
func (manager *AzureBlobStorageManager) GetObjectNameFromLocation(location string) (string, error)
GetObjectNameFromLocation gets the object name/key name from the object location url
https://account-name.blob.core.windows.net/container-name/key - >> key
func (*AzureBlobStorageManager) ListFilesWithPrefix ¶ added in v0.1.10
func (manager *AzureBlobStorageManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
func (*AzureBlobStorageManager) SetTimeout ¶ added in v0.1.10
func (manager *AzureBlobStorageManager) SetTimeout(timeout time.Duration)
func (*AzureBlobStorageManager) Upload ¶
func (manager *AzureBlobStorageManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
Upload passed in file to Azure Blob Storage
type DOSpacesConfig ¶ added in v0.1.10
type DOSpacesConfig struct { Bucket string Prefix string EndPoint string AccessKeyID string AccessKey string Region *string ForcePathStyle *bool DisableSSL *bool ContinuationToken *string IsTruncated bool }
func GetDOSpacesConfig ¶ added in v0.1.10
func GetDOSpacesConfig(config map[string]interface{}) *DOSpacesConfig
type DOSpacesManager ¶ added in v0.1.10
type DOSpacesManager struct { Config *DOSpacesConfig // contains filtered or unexported fields }
func (*DOSpacesManager) DeleteObjects ¶ added in v0.1.10
func (manager *DOSpacesManager) DeleteObjects(ctx context.Context, keys []string) error
func (*DOSpacesManager) GetConfiguredPrefix ¶ added in v0.1.10
func (manager *DOSpacesManager) GetConfiguredPrefix() string
func (*DOSpacesManager) GetDownloadKeyFromFileLocation ¶ added in v0.1.10
func (manager *DOSpacesManager) GetDownloadKeyFromFileLocation(location string) string
func (*DOSpacesManager) GetObjectNameFromLocation ¶ added in v0.1.10
func (manager *DOSpacesManager) GetObjectNameFromLocation(location string) (string, error)
GetObjectNameFromLocation gets the object name/key name from the object location url
https://rudder.sgp1.digitaloceanspaces.com/key - >> key
func (*DOSpacesManager) ListFilesWithPrefix ¶ added in v0.1.10
func (manager *DOSpacesManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
func (*DOSpacesManager) SetTimeout ¶ added in v0.1.10
func (manager *DOSpacesManager) SetTimeout(timeout time.Duration)
func (*DOSpacesManager) Upload ¶ added in v0.1.10
func (manager *DOSpacesManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
Upload passed in file to spaces
type FileManager ¶
type FileManager interface { Upload(context.Context, *os.File, ...string) (UploadOutput, error) Download(context.Context, *os.File, string) error GetObjectNameFromLocation(string) (string, error) GetDownloadKeyFromFileLocation(location string) string DeleteObjects(ctx context.Context, keys []string) error ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error) GetConfiguredPrefix() string SetTimeout(timeout time.Duration) }
FileManager implements all upload methods
type FileManagerFactory ¶ added in v0.1.10
type FileManagerFactory interface {
New(settings *SettingsT) (FileManager, error)
}
var ( DefaultFileManagerFactory FileManagerFactory ErrKeyNotFound = errors.New("NoSuchKey") )
type FileManagerFactoryT ¶ added in v0.1.10
type FileManagerFactoryT struct{}
func (*FileManagerFactoryT) New ¶ added in v0.1.10
func (*FileManagerFactoryT) New(settings *SettingsT) (FileManager, error)
New returns FileManager backed by configured provider
type FileObject ¶ added in v0.1.10
type GCSConfig ¶
type GCSConfig struct { Bucket string Prefix string Credentials string EndPoint *string ForcePathStyle *bool DisableSSL *bool Iterator *storage.ObjectIterator }
func GetGCSConfig ¶
type GCSManager ¶
type GCSManager struct { Config *GCSConfig // contains filtered or unexported fields }
func (*GCSManager) DeleteObjects ¶ added in v0.1.10
func (*GCSManager) DeleteObjects(_ context.Context, _ []string) (err error)
func (*GCSManager) GetConfiguredPrefix ¶ added in v0.1.10
func (manager *GCSManager) GetConfiguredPrefix() string
func (*GCSManager) GetDownloadKeyFromFileLocation ¶ added in v0.1.10
func (manager *GCSManager) GetDownloadKeyFromFileLocation(location string) string
TODO complete this
func (*GCSManager) GetObjectNameFromLocation ¶ added in v0.1.10
func (manager *GCSManager) GetObjectNameFromLocation(location string) (string, error)
GetObjectNameFromLocation gets the object name/key name from the object location url
https://storage.googleapis.com/bucket-name/key - >> key
func (*GCSManager) ListFilesWithPrefix ¶ added in v0.1.10
func (manager *GCSManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
func (*GCSManager) SetTimeout ¶ added in v0.1.10
func (manager *GCSManager) SetTimeout(timeout time.Duration)
func (*GCSManager) Upload ¶
func (manager *GCSManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
type MinioConfig ¶
type MinioConfig struct { Bucket string Prefix string EndPoint string AccessKeyID string SecretAccessKey string UseSSL bool ContinuationToken string IsTruncated bool }
func GetMinioConfig ¶
func GetMinioConfig(config map[string]interface{}) *MinioConfig
type MinioManager ¶
type MinioManager struct { Config *MinioConfig // contains filtered or unexported fields }
func (*MinioManager) DeleteObjects ¶ added in v0.1.10
func (manager *MinioManager) DeleteObjects(ctx context.Context, keys []string) (err error)
func (*MinioManager) GetConfiguredPrefix ¶ added in v0.1.10
func (manager *MinioManager) GetConfiguredPrefix() string
func (*MinioManager) GetDownloadKeyFromFileLocation ¶ added in v0.1.10
func (manager *MinioManager) GetDownloadKeyFromFileLocation(location string) string
func (*MinioManager) GetObjectNameFromLocation ¶ added in v0.1.10
func (manager *MinioManager) GetObjectNameFromLocation(location string) (string, error)
GetObjectNameFromLocation gets the object name/key name from the object location url
https://minio-endpoint/bucket-name/key1 - >> key1 http://minio-endpoint/bucket-name/key2 - >> key2
func (*MinioManager) ListFilesWithPrefix ¶ added in v0.1.10
func (manager *MinioManager) ListFilesWithPrefix(_ context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
func (*MinioManager) ObjectUrl ¶
func (manager *MinioManager) ObjectUrl(objectName string) string
func (*MinioManager) SetTimeout ¶ added in v0.1.10
func (manager *MinioManager) SetTimeout(timeout time.Duration)
func (*MinioManager) Upload ¶
func (manager *MinioManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)
type ObjectIterator ¶ added in v1.2.0
type ObjectIterator struct {
// contains filtered or unexported fields
}
func IterateFilesWithPrefix ¶ added in v1.2.0
func IterateFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64, manager *FileManager) *ObjectIterator
func (*ObjectIterator) Err ¶ added in v1.2.0
func (it *ObjectIterator) Err() error
func (*ObjectIterator) Get ¶ added in v1.2.0
func (it *ObjectIterator) Get() *FileObject
func (*ObjectIterator) Next ¶ added in v1.2.0
func (it *ObjectIterator) Next() bool
type S3Config ¶
type S3Config struct { Bucket string `mapstructure:"bucketName"` Prefix string `mapstructure:"Prefix"` Region *string `mapstructure:"region"` AccessKeyID string `mapstructure:"accessKeyID"` AccessKey string `mapstructure:"accessKey"` IAMRoleARN string `mapstructure:"iamRoleARN"` ExternalID string `mapstructure:"externalID"` Endpoint *string `mapstructure:"endpoint"` S3ForcePathStyle *bool `mapstructure:"s3ForcePathStyle"` DisableSSL *bool `mapstructure:"disableSSL"` EnableSSE bool `mapstructure:"enableSSE"` RegionHint string `mapstructure:"regionHint"` ContinuationToken *string `mapstructure:"continuationToken"` IsTruncated bool `mapstructure:"isTruncated"` UseGlue bool `mapstructure:"useGlue"` }
func GetS3Config ¶
type S3Manager ¶
type S3Manager struct { Config *S3Config // contains filtered or unexported fields }
func (*S3Manager) DeleteObjects ¶ added in v0.1.10
func (*S3Manager) GetConfiguredPrefix ¶ added in v0.1.10
func (*S3Manager) GetDownloadKeyFromFileLocation ¶ added in v0.1.10
func (*S3Manager) GetObjectNameFromLocation ¶ added in v0.1.10
GetObjectNameFromLocation gets the object name/key name from the object location url
https://bucket-name.s3.amazonaws.com/key - >> key
func (*S3Manager) ListFilesWithPrefix ¶
func (manager *S3Manager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
IMPT NOTE: `ListFilesWithPrefix` support Continuation Token. So, if you want same set of files (says 1st 1000 again) then create a new S3Manager & not use the existing one. Since, using the existing one will by default return next 1000 files.