storage

package
v2.6.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 4, 2024 License: MIT Imports: 61 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// BufferSize - size of ring buffer between stream handlers
	BufferSize = 128 * 1024
)

Variables

View Source
var (
	// ErrNotFound is returned when file/object cannot be found
	ErrNotFound = errors.New("key not found")
)

Functions

func AdjustAzblobBufferSize added in v2.6.0

func AdjustAzblobBufferSize(bufferSize int) int

func AdjustS3PartSize added in v2.6.0

func AdjustS3PartSize(partSize, minSize int64) int64

Types

type AzureBlob

type AzureBlob struct {
	Container azblob.ContainerURL
	Pipeline  pipeline.Pipeline
	CPK       azblob.ClientProvidedKeyOptions
	Config    *config.AzureBlobConfig
}

AzureBlob - presents methods for manipulate data on Azure

func (*AzureBlob) Close

func (a *AzureBlob) Close(ctx context.Context) error

func (*AzureBlob) Connect

func (a *AzureBlob) Connect(ctx context.Context) error

Connect - connect to Azure

func (*AzureBlob) CopyObject

func (a *AzureBlob) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*AzureBlob) DeleteFile

func (a *AzureBlob) DeleteFile(ctx context.Context, key string) error

func (*AzureBlob) DeleteFileFromObjectDiskBackup

func (a *AzureBlob) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*AzureBlob) GetFileReader

func (a *AzureBlob) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderAbsolute added in v2.5.0

func (a *AzureBlob) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderWithLocalPath

func (a *AzureBlob) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*AzureBlob) Kind

func (a *AzureBlob) Kind() string

func (*AzureBlob) PutFile

func (a *AzureBlob) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*AzureBlob) PutFileAbsolute added in v2.5.0

func (a *AzureBlob) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*AzureBlob) StatFile

func (a *AzureBlob) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*AzureBlob) Walk

func (a *AzureBlob) Walk(ctx context.Context, azPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*AzureBlob) WalkAbsolute added in v2.4.27

func (a *AzureBlob) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type Backup

type Backup struct {
	metadata.BackupMetadata
	Broken     string
	UploadDate time.Time `json:"upload_date"`
}

func GetBackupsToDeleteRemote added in v2.4.28

func GetBackupsToDeleteRemote(backups []Backup, keep int) []Backup

type BackupDestination

type BackupDestination struct {
	RemoteStorage
	// contains filtered or unexported fields
}

func NewBackupDestination

func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhouse.ClickHouse, backupName string) (*BackupDestination, error)

func (*BackupDestination) BackupList

func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, parseMetadataOnly string) ([]Backup, error)

func (*BackupDestination) DownloadCompressedStream

func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remotePath string, localPath string, maxSpeed uint64) error

func (*BackupDestination) DownloadPath

func (bd *BackupDestination) DownloadPath(ctx context.Context, remotePath string, localPath string, RetriesOnFailure int, RetriesDuration time.Duration, maxSpeed uint64) error

func (*BackupDestination) RemoveBackupRemote added in v2.4.28

func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup, cfg *config.Config) error

func (*BackupDestination) UploadCompressedStream

func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLocalPath string, files []string, remotePath string, maxSpeed uint64) error

func (*BackupDestination) UploadPath

func (bd *BackupDestination) UploadPath(ctx context.Context, baseLocalPath string, files []string, remotePath string, RetriesOnFailure int, RetriesDuration time.Duration, maxSpeed uint64) (int64, error)

type COS

type COS struct {
	Config *config.COSConfig
	// contains filtered or unexported fields
}

func (*COS) Close

func (c *COS) Close(ctx context.Context) error

func (*COS) Connect

func (c *COS) Connect(ctx context.Context) error

Connect - connect to cos

func (*COS) CopyObject

func (c *COS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*COS) DeleteFile

func (c *COS) DeleteFile(ctx context.Context, key string) error

func (*COS) DeleteFileFromObjectDiskBackup

func (c *COS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*COS) GetFileReader

func (c *COS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderAbsolute added in v2.5.0

func (c *COS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderWithLocalPath

func (c *COS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*COS) Kind

func (c *COS) Kind() string

func (*COS) PutFile

func (c *COS) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*COS) PutFileAbsolute added in v2.5.0

func (c *COS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*COS) StatFile

func (c *COS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*COS) Walk

func (c *COS) Walk(ctx context.Context, cosPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*COS) WalkAbsolute added in v2.4.27

func (c *COS) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTP

type FTP struct {
	Config *config.FTPConfig
	// contains filtered or unexported fields
}

func (*FTP) Close

func (f *FTP) Close(ctx context.Context) error

func (*FTP) Connect

func (f *FTP) Connect(ctx context.Context) error

func (*FTP) CopyObject

func (f *FTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*FTP) DeleteFile

func (f *FTP) DeleteFile(ctx context.Context, key string) error

func (*FTP) DeleteFileFromObjectDiskBackup

func (f *FTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*FTP) GetFileReader

func (f *FTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderAbsolute added in v2.5.0

func (f *FTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderWithLocalPath

func (f *FTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*FTP) Kind

func (f *FTP) Kind() string

func (*FTP) MkdirAll

func (f *FTP) MkdirAll(key string, client *ftp.ServerConn) error

func (*FTP) PutFile

func (f *FTP) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*FTP) PutFileAbsolute added in v2.5.0

func (f *FTP) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*FTP) StatFile

func (f *FTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*FTP) Walk

func (f *FTP) Walk(ctx context.Context, ftpPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*FTP) WalkAbsolute added in v2.4.27

func (f *FTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTPFileReader

type FTPFileReader struct {
	*ftp.Response
	// contains filtered or unexported fields
}

func (*FTPFileReader) Close

func (fr *FTPFileReader) Close() error

type GCS

type GCS struct {
	Config *config.GCSConfig
	// contains filtered or unexported fields
}

GCS - presents methods for manipulate data on GCS

func (*GCS) Close

func (gcs *GCS) Close(ctx context.Context) error

func (*GCS) Connect

func (gcs *GCS) Connect(ctx context.Context) error

Connect - connect to GCS

func (*GCS) CopyObject

func (gcs *GCS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*GCS) DeleteFile

func (gcs *GCS) DeleteFile(ctx context.Context, key string) error

func (*GCS) DeleteFileFromObjectDiskBackup

func (gcs *GCS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*GCS) GetFileReader

func (gcs *GCS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderAbsolute added in v2.5.0

func (gcs *GCS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderWithLocalPath

func (gcs *GCS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*GCS) Kind

func (gcs *GCS) Kind() string

func (*GCS) PutFile

func (gcs *GCS) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*GCS) PutFileAbsolute added in v2.5.0

func (gcs *GCS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*GCS) StatFile

func (gcs *GCS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*GCS) Walk

func (gcs *GCS) Walk(ctx context.Context, gcsPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*GCS) WalkAbsolute added in v2.4.27

func (gcs *GCS) WalkAbsolute(ctx context.Context, rootPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type RecalculateV4Signature

type RecalculateV4Signature struct {
	// contains filtered or unexported fields
}

RecalculateV4Signature allow GCS over S3, remove Accept-Encoding header from sign https://stackoverflow.com/a/74382598/1204665, https://github.com/aws/aws-sdk-go-v2/issues/1816

func (*RecalculateV4Signature) RoundTrip

func (lt *RecalculateV4Signature) RoundTrip(req *http.Request) (*http.Response, error)

type RemoteFile

type RemoteFile interface {
	Size() int64
	Name() string
	LastModified() time.Time
}

RemoteFile - interface describe file on remote storage

type RemoteStorage

type RemoteStorage interface {
	Kind() string
	Connect(ctx context.Context) error
	Close(ctx context.Context) error
	StatFile(ctx context.Context, key string) (RemoteFile, error)
	DeleteFile(ctx context.Context, key string) error
	DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
	Walk(ctx context.Context, prefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	WalkAbsolute(ctx context.Context, absolutePrefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error)
	PutFile(ctx context.Context, key string, r io.ReadCloser) error
	PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error
	CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
}

RemoteStorage -

type S3

type S3 struct {
	Config      *config.S3Config
	PartSize    int64
	Concurrency int
	BufferSize  int
	// contains filtered or unexported fields
}

S3 - presents methods for manipulate data on s3

func (*S3) Close

func (s *S3) Close(ctx context.Context) error

func (*S3) Connect

func (s *S3) Connect(ctx context.Context) error

Connect - connect to s3

func (*S3) CopyObject

func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*S3) DeleteFile

func (s *S3) DeleteFile(ctx context.Context, key string) error

func (*S3) DeleteFileFromObjectDiskBackup

func (s *S3) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*S3) GetFileReader

func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderAbsolute added in v2.5.0

func (s *S3) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderWithLocalPath

func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error)

func (*S3) Kind

func (s *S3) Kind() string

func (*S3) PutFile

func (s *S3) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*S3) PutFileAbsolute added in v2.5.0

func (s *S3) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*S3) ResolveEndpoint added in v2.6.0

func (s *S3) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (endpoint smithyendpoints.Endpoint, err error)

func (*S3) StatFile

func (s *S3) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*S3) Walk

func (s *S3) Walk(ctx context.Context, s3Path string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*S3) WalkAbsolute added in v2.4.27

func (s *S3) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type S3LogToZeroLogAdapter added in v2.5.22

type S3LogToZeroLogAdapter struct {
	// contains filtered or unexported fields
}

func (S3LogToZeroLogAdapter) Logf added in v2.5.22

func (S3LogToApexLogAdapter S3LogToZeroLogAdapter) Logf(severity awsV2Logging.Classification, msg string, args ...interface{})

type SFTP

type SFTP struct {
	Config *config.SFTPConfig
	// contains filtered or unexported fields
}

SFTP Implement RemoteStorage

func (*SFTP) Close

func (sftp *SFTP) Close(ctx context.Context) error

func (*SFTP) Connect

func (sftp *SFTP) Connect(ctx context.Context) error

func (*SFTP) CopyObject

func (sftp *SFTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*SFTP) Debug

func (sftp *SFTP) Debug(msg string, v ...interface{})

func (*SFTP) DeleteDirectory

func (sftp *SFTP) DeleteDirectory(ctx context.Context, dirPath string) error

func (*SFTP) DeleteFile

func (sftp *SFTP) DeleteFile(ctx context.Context, key string) error

func (*SFTP) DeleteFileFromObjectDiskBackup

func (sftp *SFTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*SFTP) GetFileReader

func (sftp *SFTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderAbsolute added in v2.5.0

func (sftp *SFTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderWithLocalPath

func (sftp *SFTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*SFTP) Kind

func (sftp *SFTP) Kind() string

func (*SFTP) PutFile

func (sftp *SFTP) PutFile(ctx context.Context, key string, localFile io.ReadCloser) error

func (*SFTP) PutFileAbsolute added in v2.5.0

func (sftp *SFTP) PutFileAbsolute(ctx context.Context, key string, localFile io.ReadCloser) error

func (*SFTP) StatFile

func (sftp *SFTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*SFTP) Walk

func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*SFTP) WalkAbsolute added in v2.4.27

func (sftp *SFTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

Directories

Path Synopsis
Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK
Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL