filemanager

package
v1.7.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 4, 2023 License: AGPL-3.0 Imports: 31 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetProviderConfigForBackupsFromEnv added in v1.0.2 

func GetProviderConfigForBackupsFromEnv(ctx context.Context) map[string]interface{}

GetProviderConfigForBackupsFromEnv returns the provider config

func GetProviderConfigFromEnv 

func GetProviderConfigFromEnv(ctx context.Context, provider string) map[string]interface{}

Types

type AzureBlobStorageConfig 

type AzureBlobStorageConfig struct {
	Container      string
	Prefix         string
	AccountName    string
	AccountKey     string
	SASToken       string
	EndPoint       *string
	ForcePathStyle *bool
	DisableSSL     *bool
	Marker         azblob.Marker
	UseSASTokens   bool
}

func GetAzureBlogStorageConfig 

func GetAzureBlogStorageConfig(config map[string]interface{}) *AzureBlobStorageConfig

type AzureBlobStorageManager 

type AzureBlobStorageManager struct {
	Config *AzureBlobStorageConfig
	// contains filtered or unexported fields
}

func (*AzureBlobStorageManager) DeleteObjects added in v0.1.10 

func (manager *AzureBlobStorageManager) DeleteObjects(ctx context.Context, keys []string) (err error)

func (*AzureBlobStorageManager) Download 

func (manager *AzureBlobStorageManager) Download(ctx context.Context, output *os.File, key string) error

func (*AzureBlobStorageManager) GetConfiguredPrefix added in v0.1.10 

func (manager *AzureBlobStorageManager) GetConfiguredPrefix() string

func (*AzureBlobStorageManager) GetDownloadKeyFromFileLocation added in v0.1.10 

func (manager *AzureBlobStorageManager) GetDownloadKeyFromFileLocation(location string) string

func (*AzureBlobStorageManager) GetObjectNameFromLocation added in v0.1.10 

func (manager *AzureBlobStorageManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://account-name.blob.core.windows.net/container-name/key - >> key

func (*AzureBlobStorageManager) ListFilesWithPrefix added in v0.1.10 

func (manager *AzureBlobStorageManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*AzureBlobStorageManager) SetTimeout added in v0.1.10 

func (manager *AzureBlobStorageManager) SetTimeout(timeout time.Duration)

func (*AzureBlobStorageManager) Upload 

func (manager *AzureBlobStorageManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

Upload passed in file to Azure Blob Storage

type DOSpacesConfig added in v0.1.10 

type DOSpacesConfig struct {
	Bucket            string
	Prefix            string
	EndPoint          string
	AccessKeyID       string
	AccessKey         string
	Region            *string
	ForcePathStyle    *bool
	DisableSSL        *bool
	ContinuationToken *string
	IsTruncated       bool
}

func GetDOSpacesConfig added in v0.1.10 

func GetDOSpacesConfig(config map[string]interface{}) *DOSpacesConfig

type DOSpacesManager added in v0.1.10 

type DOSpacesManager struct {
	Config *DOSpacesConfig
	// contains filtered or unexported fields
}

func (*DOSpacesManager) DeleteObjects added in v0.1.10 

func (manager *DOSpacesManager) DeleteObjects(ctx context.Context, keys []string) error

func (*DOSpacesManager) Download added in v0.1.10 

func (manager *DOSpacesManager) Download(ctx context.Context, output *os.File, key string) error

func (*DOSpacesManager) GetConfiguredPrefix added in v0.1.10 

func (manager *DOSpacesManager) GetConfiguredPrefix() string

func (*DOSpacesManager) GetDownloadKeyFromFileLocation added in v0.1.10 

func (manager *DOSpacesManager) GetDownloadKeyFromFileLocation(location string) string

func (*DOSpacesManager) GetObjectNameFromLocation added in v0.1.10 

func (manager *DOSpacesManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://rudder.sgp1.digitaloceanspaces.com/key - >> key

func (*DOSpacesManager) ListFilesWithPrefix added in v0.1.10 

func (manager *DOSpacesManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*DOSpacesManager) SetTimeout added in v0.1.10 

func (manager *DOSpacesManager) SetTimeout(timeout time.Duration)

func (*DOSpacesManager) Upload added in v0.1.10 

func (manager *DOSpacesManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

Upload passed in file to spaces

type FileManager 

type FileManager interface {
	Upload(context.Context, *os.File, ...string) (UploadOutput, error)
	Download(context.Context, *os.File, string) error
	GetObjectNameFromLocation(string) (string, error)
	GetDownloadKeyFromFileLocation(location string) string
	DeleteObjects(ctx context.Context, keys []string) error
	ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)
	GetConfiguredPrefix() string
	SetTimeout(timeout time.Duration)
}

FileManager implements all upload methods

type FileManagerFactory added in v0.1.10 

type FileManagerFactory interface {
	New(settings *SettingsT) (FileManager, error)
}
var (
	DefaultFileManagerFactory FileManagerFactory
	ErrKeyNotFound            = errors.New("NoSuchKey")
)

type FileManagerFactoryT added in v0.1.10 

type FileManagerFactoryT struct{}

func (*FileManagerFactoryT) New added in v0.1.10 

func (*FileManagerFactoryT) New(settings *SettingsT) (FileManager, error)

New returns FileManager backed by configured provider

type FileObject added in v0.1.10 

type FileObject struct {
	Key          string
	LastModified time.Time
}

type GCSConfig 

type GCSConfig struct {
	Bucket         string
	Prefix         string
	Credentials    string
	EndPoint       *string
	ForcePathStyle *bool
	DisableSSL     *bool
	Iterator       *storage.ObjectIterator
}

func GetGCSConfig 

func GetGCSConfig(config map[string]interface{}) *GCSConfig

type GCSManager 

type GCSManager struct {
	Config *GCSConfig
	// contains filtered or unexported fields
}

func (*GCSManager) DeleteObjects added in v0.1.10 

func (*GCSManager) DeleteObjects(_ context.Context, _ []string) (err error)

func (*GCSManager) Download 

func (manager *GCSManager) Download(ctx context.Context, output *os.File, key string) error

func (*GCSManager) GetConfiguredPrefix added in v0.1.10 

func (manager *GCSManager) GetConfiguredPrefix() string

func (*GCSManager) GetDownloadKeyFromFileLocation added in v0.1.10 

func (manager *GCSManager) GetDownloadKeyFromFileLocation(location string) string

TODO complete this

func (*GCSManager) GetObjectNameFromLocation added in v0.1.10 

func (manager *GCSManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://storage.googleapis.com/bucket-name/key - >> key

func (*GCSManager) ListFilesWithPrefix added in v0.1.10 

func (manager *GCSManager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*GCSManager) SetTimeout added in v0.1.10 

func (manager *GCSManager) SetTimeout(timeout time.Duration)

func (*GCSManager) Upload 

func (manager *GCSManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

type MinioConfig 

type MinioConfig struct {
	Bucket            string
	Prefix            string
	EndPoint          string
	AccessKeyID       string
	SecretAccessKey   string
	UseSSL            bool
	ContinuationToken string
	IsTruncated       bool
}

func GetMinioConfig 

func GetMinioConfig(config map[string]interface{}) *MinioConfig

type MinioManager 

type MinioManager struct {
	Config *MinioConfig
	// contains filtered or unexported fields
}

func (*MinioManager) DeleteObjects added in v0.1.10 

func (manager *MinioManager) DeleteObjects(ctx context.Context, keys []string) (err error)

func (*MinioManager) Download 

func (manager *MinioManager) Download(ctx context.Context, file *os.File, key string) error

func (*MinioManager) GetConfiguredPrefix added in v0.1.10 

func (manager *MinioManager) GetConfiguredPrefix() string

func (*MinioManager) GetDownloadKeyFromFileLocation added in v0.1.10 

func (manager *MinioManager) GetDownloadKeyFromFileLocation(location string) string

func (*MinioManager) GetObjectNameFromLocation added in v0.1.10 

func (manager *MinioManager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://minio-endpoint/bucket-name/key1 - >> key1
http://minio-endpoint/bucket-name/key2 - >> key2

func (*MinioManager) ListFilesWithPrefix added in v0.1.10 

func (manager *MinioManager) ListFilesWithPrefix(_ context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

func (*MinioManager) ObjectUrl 

func (manager *MinioManager) ObjectUrl(objectName string) string

func (*MinioManager) SetTimeout added in v0.1.10 

func (manager *MinioManager) SetTimeout(timeout time.Duration)

func (*MinioManager) Upload 

func (manager *MinioManager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

type ObjectIterator added in v1.2.0 

type ObjectIterator struct {
	// contains filtered or unexported fields
}

func IterateFilesWithPrefix added in v1.2.0 

func IterateFilesWithPrefix(ctx context.Context, prefix, startAfter string, maxItems int64, manager *FileManager) *ObjectIterator

func (*ObjectIterator) Err added in v1.2.0 

func (it *ObjectIterator) Err() error

func (*ObjectIterator) Get added in v1.2.0 

func (it *ObjectIterator) Get() *FileObject

func (*ObjectIterator) Next added in v1.2.0 

func (it *ObjectIterator) Next() bool

type S3Config 

type S3Config struct {
	Bucket            string  `mapstructure:"bucketName"`
	Prefix            string  `mapstructure:"Prefix"`
	Region            *string `mapstructure:"region"`
	Endpoint          *string `mapstructure:"endpoint"`
	S3ForcePathStyle  *bool   `mapstructure:"s3ForcePathStyle"`
	DisableSSL        *bool   `mapstructure:"disableSSL"`
	EnableSSE         bool    `mapstructure:"enableSSE"`
	RegionHint        string  `mapstructure:"regionHint"`
	ContinuationToken *string `mapstructure:"continuationToken"`
	IsTruncated       bool    `mapstructure:"isTruncated"`
	UseGlue           bool    `mapstructure:"useGlue"`
}

type S3Manager 

type S3Manager struct {
	Config        *S3Config
	SessionConfig *awsutils.SessionConfig
	// contains filtered or unexported fields
}

func NewS3Manager added in v1.4.5 

func NewS3Manager(config map[string]interface{}) (*S3Manager, error)

func (*S3Manager) DeleteObjects added in v0.1.10 

func (manager *S3Manager) DeleteObjects(ctx context.Context, keys []string) (err error)

func (*S3Manager) Download 

func (manager *S3Manager) Download(ctx context.Context, output *os.File, key string) error

func (*S3Manager) GetConfiguredPrefix added in v0.1.10 

func (manager *S3Manager) GetConfiguredPrefix() string

func (*S3Manager) GetDownloadKeyFromFileLocation added in v0.1.10 

func (manager *S3Manager) GetDownloadKeyFromFileLocation(location string) string

func (*S3Manager) GetObjectNameFromLocation added in v0.1.10 

func (manager *S3Manager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url 

https://bucket-name.s3.amazonaws.com/key - >> key

func (*S3Manager) ListFilesWithPrefix 

func (manager *S3Manager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) (fileObjects []*FileObject, err error)

IMPT NOTE: `ListFilesWithPrefix` support Continuation Token. So, if you want same set of files (says 1st 1000 again) then create a new S3Manager & not use the existing one. Since, using the existing one will by default return next 1000 files.

func (*S3Manager) SetTimeout added in v0.1.10 

func (manager *S3Manager) SetTimeout(timeout time.Duration)

func (*S3Manager) Upload 

func (manager *S3Manager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadOutput, error)

Upload passed in file to s3

type SettingsT 

type SettingsT struct {
	Provider string
	Config   map[string]interface{}
}

SettingsT sets configuration for FileManager

type UploadOutput 

type UploadOutput struct {
	Location   string
	ObjectName string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.