filemanager

Documentation

Index

• Variables
• func GetProviderConfigFromEnv(opts ProviderConfigOpts) map[string]interface{}
• func NewAzureBlobManager(config map[string]interface{}, log logger.Logger, ...) (*azureBlobManager, error)
• func NewDigitalOceanManager(config map[string]interface{}, log logger.Logger, ...) (*digitalOceanManager, error)
• func NewGCSManager(config map[string]interface{}, log logger.Logger, ...) (*gcsManager, error)
• func NewMinioManager(config map[string]interface{}, log logger.Logger, ...) (*minioManager, error)
• func WithGCSUploadIfObjectNotExist(uploadIfNotExist bool) func(*GCSConfig)
• type AzureBlobConfig
• type DigitalOceanConfig
• type Factory
• type FileInfo
• type FileManager
  • func New(settings *Settings) (FileManager, error)
• type GCSConfig
• type GCSOpt
• type ListSession
  • func MockListSession(fileObjects []*FileInfo, err error) ListSession
• type MinioConfig
• type ObjectIterator
  • func IterateFilesWithPrefix(ctx context.Context, prefix, startAfter string, maxItems int64, ...) *ObjectIterator
  • func NewListIterator(session ListSession) *ObjectIterator
  • func (it *ObjectIterator) Err() error
  • func (it *ObjectIterator) Get() *FileInfo
  • func (it *ObjectIterator) Next() bool
• type ProviderConfigOpts
• type S3Config
• type S3Manager
  • func NewS3Manager(config map[string]interface{}, log logger.Logger, ...) (*S3Manager, error)
  • func (m *S3Manager) Bucket() string
  • func (m *S3Manager) Delete(ctx context.Context, keys []string) (err error)
  • func (m *S3Manager) Download(ctx context.Context, output *os.File, key string) error
  • func (m *S3Manager) GetDownloadKeyFromFileLocation(location string) string
  • func (m *S3Manager) GetObjectNameFromLocation(location string) (string, error)
  • func (m *S3Manager) GetSession(ctx context.Context) (*session.Session, error)
  • func (m *S3Manager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) ListSession
  • func (m *S3Manager) Prefix() string
  • func (manager S3Manager) SetTimeout(timeout time.Duration)
  • func (m *S3Manager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadedFile, error)
• type Settings
• type UploadedFile

Variables

var (
	ErrKeyNotFound            = errors.New("NoSuchKey")
	ErrInvalidServiceProvider = errors.New("service provider not supported")
	ErrPreConditionFailed     = errors.New("precondition failed")
)

Functions

func GetProviderConfigFromEnv

func GetProviderConfigFromEnv(opts ProviderConfigOpts) map[string]interface{}

func NewAzureBlobManager

func NewAzureBlobManager(config map[string]interface{}, log logger.Logger, defaultTimeout func() time.Duration) (*azureBlobManager, error)

NewAzureBlobManager creates a new file manager for Azure Blob Storage

func NewDigitalOceanManager

func NewDigitalOceanManager(config map[string]interface{}, log logger.Logger, defaultTimeout func() time.Duration) (*digitalOceanManager, error)

NewDigitalOceanManager creates a new file manager for digital ocean spaces

func NewGCSManager

func NewGCSManager(
	config map[string]interface{}, log logger.Logger, defaultTimeout func() time.Duration, opts ...GCSOpt,
) (*gcsManager, error)

NewGCSManager creates a new file manager for Google Cloud Storage

func NewMinioManager

func NewMinioManager(config map[string]interface{}, log logger.Logger, defaultTimeout func() time.Duration) (*minioManager, error)

NewMinioManager creates a new file manager for minio

func WithGCSUploadIfObjectNotExist

func WithGCSUploadIfObjectNotExist(uploadIfNotExist bool) func(*GCSConfig)

Types

type AzureBlobConfig

type AzureBlobConfig struct {
	Container      string
	Prefix         string
	AccountName    string
	AccountKey     string
	SASToken       string
	EndPoint       *string
	ForcePathStyle *bool
	DisableSSL     *bool
	UseSASTokens   bool
}

type DigitalOceanConfig

type DigitalOceanConfig struct {
	Bucket         string
	Prefix         string
	EndPoint       string
	AccessKeyID    string
	AccessKey      string
	Region         *string
	ForcePathStyle *bool
	DisableSSL     *bool
}

type Factory

type Factory func(settings *Settings) (FileManager, error)

Factory is a function that returns a new file manager

type FileInfo

type FileInfo struct {
	Key          string
	LastModified time.Time
}

FileInfo contains information about a file

type FileManager

type FileManager interface {
	// ListFilesWithPrefix starts a list session for files with given prefix
	ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) ListSession
	// Download downloads the file with given key to the passed in file
	Download(context.Context, *os.File, string) error
	// Upload uploads the passed in file to the file manager
	Upload(context.Context, *os.File, ...string) (UploadedFile, error)
	// Delete deletes the file(s) with given key(s)
	Delete(ctx context.Context, keys []string) error

	// Prefix returns the prefix for the file manager
	Prefix() string
	// SetTimeout overrides the default timeout for the file manager
	SetTimeout(timeout time.Duration)

	// GetObjectNameFromLocation gets the object name/key name from the object location url
	GetObjectNameFromLocation(string) (string, error)
	// GetDownloadKeyFromFileLocation gets the download key from the object location url
	GetDownloadKeyFromFileLocation(string) string
}

FileManager is able to manage files in a storage provider

func New

func New(settings *Settings) (FileManager, error)

New returns file manager backed by configured provider

type GCSConfig

type GCSConfig struct {
	Bucket         string
	Prefix         string
	Credentials    string
	EndPoint       *string
	ForcePathStyle *bool
	DisableSSL     *bool
	JSONReads      bool
	// contains filtered or unexported fields
}

type GCSOpt

type GCSOpt func(*GCSConfig)

type ListSession

type ListSession interface {
	// Next returns the next batch of files, until there are no more files for this session
	Next() (fileObjects []*FileInfo, err error)
}

ListSession is a session for listing files

func MockListSession

func MockListSession(fileObjects []*FileInfo, err error) ListSession

MockListSession is a mock implementation of ListSession that always returns the given file objects and error

type MinioConfig

type MinioConfig struct {
	Bucket          string
	Prefix          string
	EndPoint        string
	AccessKeyID     string
	SecretAccessKey string
	UseSSL          bool
}

type ObjectIterator

type ObjectIterator struct {
	// contains filtered or unexported fields
}

func IterateFilesWithPrefix

func IterateFilesWithPrefix(ctx context.Context, prefix, startAfter string, maxItems int64, manager FileManager) *ObjectIterator

IterateFilesWithPrefix returns an iterator that can be used to iterate over all files with the given prefix.

func NewListIterator

func NewListIterator(session ListSession) *ObjectIterator

NewListIterator returns a new iterator for the given list session.

func (*ObjectIterator) Err

func (it *ObjectIterator) Err() error

func (*ObjectIterator) Get

func (it *ObjectIterator) Get() *FileInfo

func (*ObjectIterator) Next

func (it *ObjectIterator) Next() bool

type ProviderConfigOpts

type ProviderConfigOpts struct {
	Provider           string
	Bucket             string
	Prefix             string
	Config             *config.Config
	ExternalIDSupplier func() string
}

type S3Config

type S3Config struct {
	Bucket           string  `mapstructure:"bucketName"`
	Prefix           string  `mapstructure:"Prefix"`
	Region           *string `mapstructure:"region"`
	Endpoint         *string `mapstructure:"endpoint"`
	S3ForcePathStyle *bool   `mapstructure:"s3ForcePathStyle"`
	DisableSSL       *bool   `mapstructure:"disableSSL"`
	EnableSSE        bool    `mapstructure:"enableSSE"`
	RegionHint       string  `mapstructure:"regionHint"`
	UseGlue          bool    `mapstructure:"useGlue"`
}

type S3Manager

type S3Manager struct {
	// contains filtered or unexported fields
}

func NewS3Manager

func NewS3Manager(
	config map[string]interface{}, log logger.Logger, defaultTimeout func() time.Duration,
) (*S3Manager, error)

NewS3Manager creates a new file manager for S3

func (*S3Manager) Bucket

func (m *S3Manager) Bucket() string

func (*S3Manager) Delete

func (m *S3Manager) Delete(ctx context.Context, keys []string) (err error)

func (*S3Manager) Download

func (m *S3Manager) Download(ctx context.Context, output *os.File, key string) error

Download downloads a file from S3

func (*S3Manager) GetDownloadKeyFromFileLocation

func (m *S3Manager) GetDownloadKeyFromFileLocation(location string) string

func (*S3Manager) GetObjectNameFromLocation

func (m *S3Manager) GetObjectNameFromLocation(location string) (string, error)

GetObjectNameFromLocation gets the object name/key name from the object location url

https://bucket-name.s3.amazonaws.com/key - >> key

func (*S3Manager) GetSession

func (m *S3Manager) GetSession(ctx context.Context) (*session.Session, error)

func (*S3Manager) ListFilesWithPrefix

func (m *S3Manager) ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) ListSession

func (*S3Manager) Prefix

func (m *S3Manager) Prefix() string

func (S3Manager) SetTimeout

func (manager S3Manager) SetTimeout(timeout time.Duration)

func (*S3Manager) Upload

func (m *S3Manager) Upload(ctx context.Context, file *os.File, prefixes ...string) (UploadedFile, error)

Upload uploads a file to S3

type Settings

type Settings struct {
	Provider string
	Config   map[string]interface{}
	Logger   logger.Logger
	Conf     *config.Config

	// when GCSUploadIfNotExist is set to true, the client uploads to GCS storage
	// only if a file with the same name doesn't exist already
	GCSUploadIfNotExist bool
}

Settings for file manager

type UploadedFile

type UploadedFile struct {
	Location   string
	ObjectName string
}

UploadedFile contains information about the uploaded file