storage

package
v2.6.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 9, 2024 License: MIT Imports: 61 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// BufferSize - size of ring buffer between stream handlers
	BufferSize = 128 * 1024
)

Variables

View Source 
var (
	// ErrNotFound is returned when file/object cannot be found
	ErrNotFound = errors.New("key not found")
)

Functions

func AdjustAzblobBufferSize added in v2.6.0 

func AdjustAzblobBufferSize(bufferSize int) int

func AdjustS3PartSize added in v2.6.0 

func AdjustS3PartSize(partSize, minSize int64) int64

Types

type AzureBlob 

type AzureBlob struct {
	Container azblob.ContainerURL
	Pipeline  pipeline.Pipeline
	CPK       azblob.ClientProvidedKeyOptions
	Config    *config.AzureBlobConfig
}

AzureBlob - presents methods for manipulate data on Azure

func (*AzureBlob) Close 

func (a *AzureBlob) Close(ctx context.Context) error

func (*AzureBlob) Connect 

func (a *AzureBlob) Connect(ctx context.Context) error

Connect - connect to Azure

func (*AzureBlob) CopyObject 

func (a *AzureBlob) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*AzureBlob) DeleteFile 

func (a *AzureBlob) DeleteFile(ctx context.Context, key string) error

func (*AzureBlob) DeleteFileFromObjectDiskBackup 

func (a *AzureBlob) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*AzureBlob) GetFileReader 

func (a *AzureBlob) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderAbsolute added in v2.5.0 

func (a *AzureBlob) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderWithLocalPath 

func (a *AzureBlob) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*AzureBlob) Kind 

func (a *AzureBlob) Kind() string

func (*AzureBlob) PutFile 

func (a *AzureBlob) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*AzureBlob) PutFileAbsolute added in v2.5.0 

func (a *AzureBlob) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*AzureBlob) StatFile 

func (a *AzureBlob) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*AzureBlob) Walk 

func (a *AzureBlob) Walk(ctx context.Context, azPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*AzureBlob) WalkAbsolute added in v2.4.27 

func (a *AzureBlob) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type Backup 

type Backup struct {
	metadata.BackupMetadata
	Broken     string
	UploadDate time.Time `json:"upload_date"`
}

func GetBackupsToDeleteRemote added in v2.4.28 

func GetBackupsToDeleteRemote(backups []Backup, keep int) []Backup

type BackupDestination 

type BackupDestination struct {
	RemoteStorage
	// contains filtered or unexported fields
}

func NewBackupDestination 

func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhouse.ClickHouse, backupName string) (*BackupDestination, error)

func (*BackupDestination) BackupList 

func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, parseMetadataOnly string) ([]Backup, error)

func (*BackupDestination) DownloadCompressedStream 

func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remotePath string, localPath string, maxSpeed uint64) error

func (*BackupDestination) DownloadPath 

func (bd *BackupDestination) DownloadPath(ctx context.Context, remotePath string, localPath string, RetriesOnFailure int, RetriesDuration time.Duration, maxSpeed uint64) error

func (*BackupDestination) RemoveBackupRemote added in v2.4.28 

func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup, cfg *config.Config) error

func (*BackupDestination) UploadCompressedStream 

func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLocalPath string, files []string, remotePath string, maxSpeed uint64) error

func (*BackupDestination) UploadPath 

func (bd *BackupDestination) UploadPath(ctx context.Context, baseLocalPath string, files []string, remotePath string, RetriesOnFailure int, RetriesDuration time.Duration, maxSpeed uint64) (int64, error)

type COS 

type COS struct {
	Config *config.COSConfig
	// contains filtered or unexported fields
}

func (*COS) Close 

func (c *COS) Close(ctx context.Context) error

func (*COS) Connect 

func (c *COS) Connect(ctx context.Context) error

Connect - connect to cos

func (*COS) CopyObject 

func (c *COS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*COS) DeleteFile 

func (c *COS) DeleteFile(ctx context.Context, key string) error

func (*COS) DeleteFileFromObjectDiskBackup 

func (c *COS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*COS) GetFileReader 

func (c *COS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderAbsolute added in v2.5.0 

func (c *COS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderWithLocalPath 

func (c *COS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*COS) Kind 

func (c *COS) Kind() string

func (*COS) PutFile 

func (c *COS) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*COS) PutFileAbsolute added in v2.5.0 

func (c *COS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*COS) StatFile 

func (c *COS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*COS) Walk 

func (c *COS) Walk(ctx context.Context, cosPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*COS) WalkAbsolute added in v2.4.27 

func (c *COS) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTP 

type FTP struct {
	Config *config.FTPConfig
	// contains filtered or unexported fields
}

func (*FTP) Close 

func (f *FTP) Close(ctx context.Context) error

func (*FTP) Connect 

func (f *FTP) Connect(ctx context.Context) error

func (*FTP) CopyObject 

func (f *FTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*FTP) DeleteFile 

func (f *FTP) DeleteFile(ctx context.Context, key string) error

func (*FTP) DeleteFileFromObjectDiskBackup 

func (f *FTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*FTP) GetFileReader 

func (f *FTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderAbsolute added in v2.5.0 

func (f *FTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderWithLocalPath 

func (f *FTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*FTP) Kind 

func (f *FTP) Kind() string

func (*FTP) MkdirAll 

func (f *FTP) MkdirAll(key string, client *ftp.ServerConn) error

func (*FTP) PutFile 

func (f *FTP) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*FTP) PutFileAbsolute added in v2.5.0 

func (f *FTP) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*FTP) StatFile 

func (f *FTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*FTP) Walk 

func (f *FTP) Walk(ctx context.Context, ftpPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*FTP) WalkAbsolute added in v2.4.27 

func (f *FTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTPFileReader 

type FTPFileReader struct {
	*ftp.Response
	// contains filtered or unexported fields
}

func (*FTPFileReader) Close 

func (fr *FTPFileReader) Close() error

type GCS 

type GCS struct {
	Config *config.GCSConfig
	// contains filtered or unexported fields
}

GCS - presents methods for manipulate data on GCS

func (*GCS) Close 

func (gcs *GCS) Close(ctx context.Context) error

func (*GCS) Connect 

func (gcs *GCS) Connect(ctx context.Context) error

Connect - connect to GCS

func (*GCS) CopyObject 

func (gcs *GCS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*GCS) DeleteFile 

func (gcs *GCS) DeleteFile(ctx context.Context, key string) error

func (*GCS) DeleteFileFromObjectDiskBackup 

func (gcs *GCS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*GCS) GetFileReader 

func (gcs *GCS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderAbsolute added in v2.5.0 

func (gcs *GCS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderWithLocalPath 

func (gcs *GCS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*GCS) Kind 

func (gcs *GCS) Kind() string

func (*GCS) PutFile 

func (gcs *GCS) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*GCS) PutFileAbsolute added in v2.5.0 

func (gcs *GCS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*GCS) StatFile 

func (gcs *GCS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*GCS) Walk 

func (gcs *GCS) Walk(ctx context.Context, gcsPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*GCS) WalkAbsolute added in v2.4.27 

func (gcs *GCS) WalkAbsolute(ctx context.Context, rootPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type RecalculateV4Signature 

type RecalculateV4Signature struct {
	// contains filtered or unexported fields
}

RecalculateV4Signature allow GCS over S3, remove Accept-Encoding header from sign https://stackoverflow.com/a/74382598/1204665, https://github.com/aws/aws-sdk-go-v2/issues/1816

func (*RecalculateV4Signature) RoundTrip 

func (lt *RecalculateV4Signature) RoundTrip(req *http.Request) (*http.Response, error)

type RemoteFile 

type RemoteFile interface {
	Size() int64
	Name() string
	LastModified() time.Time
}

RemoteFile - interface describe file on remote storage

type RemoteStorage 

type RemoteStorage interface {
	Kind() string
	Connect(ctx context.Context) error
	Close(ctx context.Context) error
	StatFile(ctx context.Context, key string) (RemoteFile, error)
	DeleteFile(ctx context.Context, key string) error
	DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
	Walk(ctx context.Context, prefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	WalkAbsolute(ctx context.Context, absolutePrefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error)
	PutFile(ctx context.Context, key string, r io.ReadCloser) error
	PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error
	CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
}

RemoteStorage -

type S3 

type S3 struct {
	Config      *config.S3Config
	PartSize    int64
	Concurrency int
	BufferSize  int
	// contains filtered or unexported fields
}

S3 - presents methods for manipulate data on s3

func (*S3) Close 

func (s *S3) Close(ctx context.Context) error

func (*S3) Connect 

func (s *S3) Connect(ctx context.Context) error

Connect - connect to s3

func (*S3) CopyObject 

func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*S3) DeleteFile 

func (s *S3) DeleteFile(ctx context.Context, key string) error

func (*S3) DeleteFileFromObjectDiskBackup 

func (s *S3) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*S3) GetFileReader 

func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderAbsolute added in v2.5.0 

func (s *S3) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderWithLocalPath 

func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error)

func (*S3) Kind 

func (s *S3) Kind() string

func (*S3) PutFile 

func (s *S3) PutFile(ctx context.Context, key string, r io.ReadCloser) error

func (*S3) PutFileAbsolute added in v2.5.0 

func (s *S3) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error

func (*S3) ResolveEndpoint added in v2.6.0 

func (s *S3) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (endpoint smithyendpoints.Endpoint, err error)

func (*S3) StatFile 

func (s *S3) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*S3) Walk 

func (s *S3) Walk(ctx context.Context, s3Path string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*S3) WalkAbsolute added in v2.4.27 

func (s *S3) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type S3LogToZeroLogAdapter added in v2.5.22 

type S3LogToZeroLogAdapter struct {
	// contains filtered or unexported fields
}

func (S3LogToZeroLogAdapter) Logf added in v2.5.22 

func (S3LogToApexLogAdapter S3LogToZeroLogAdapter) Logf(severity awsV2Logging.Classification, msg string, args ...interface{})

type SFTP 

type SFTP struct {
	Config *config.SFTPConfig
	// contains filtered or unexported fields
}

SFTP Implement RemoteStorage

func (*SFTP) Close 

func (sftp *SFTP) Close(ctx context.Context) error

func (*SFTP) Connect 

func (sftp *SFTP) Connect(ctx context.Context) error

func (*SFTP) CopyObject 

func (sftp *SFTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*SFTP) Debug 

func (sftp *SFTP) Debug(msg string, v ...interface{})

func (*SFTP) DeleteDirectory 

func (sftp *SFTP) DeleteDirectory(ctx context.Context, dirPath string) error

func (*SFTP) DeleteFile 

func (sftp *SFTP) DeleteFile(ctx context.Context, key string) error

func (*SFTP) DeleteFileFromObjectDiskBackup 

func (sftp *SFTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*SFTP) GetFileReader 

func (sftp *SFTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderAbsolute added in v2.5.0 

func (sftp *SFTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderWithLocalPath 

func (sftp *SFTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)

func (*SFTP) Kind 

func (sftp *SFTP) Kind() string

func (*SFTP) PutFile 

func (sftp *SFTP) PutFile(ctx context.Context, key string, localFile io.ReadCloser) error

func (*SFTP) PutFileAbsolute added in v2.5.0 

func (sftp *SFTP) PutFileAbsolute(ctx context.Context, key string, localFile io.ReadCloser) error

func (*SFTP) StatFile 

func (sftp *SFTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*SFTP) Walk 

func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*SFTP) WalkAbsolute added in v2.4.27 

func (sftp *SFTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

Source Files

View all Source files

Directories

Path Synopsis
Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK
 Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.