Documentation ¶
Index ¶
- Constants
- Variables
- func AdjustAzblobBufferSize(bufferSize int) int
- func AdjustS3PartSize(partSize, minSize int64) int64
- type AzureBlob
- func (a *AzureBlob) Close(ctx context.Context) error
- func (a *AzureBlob) Connect(ctx context.Context) error
- func (a *AzureBlob) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
- func (a *AzureBlob) DeleteFile(ctx context.Context, key string) error
- func (a *AzureBlob) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
- func (a *AzureBlob) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
- func (a *AzureBlob) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
- func (a *AzureBlob) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)
- func (a *AzureBlob) Kind() string
- func (a *AzureBlob) PutFile(ctx context.Context, key string, r io.ReadCloser) error
- func (a *AzureBlob) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error
- func (a *AzureBlob) StatFile(ctx context.Context, key string) (RemoteFile, error)
- func (a *AzureBlob) Walk(ctx context.Context, azPath string, recursive bool, ...) error
- func (a *AzureBlob) WalkAbsolute(ctx context.Context, prefix string, recursive bool, ...) error
- type Backup
- type BackupDestination
- func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, parseMetadataOnly string) ([]Backup, error)
- func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remotePath string, localPath string, maxSpeed uint64) error
- func (bd *BackupDestination) DownloadPath(ctx context.Context, remotePath string, localPath string, RetriesOnFailure int, ...) error
- func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup, cfg *config.Config) error
- func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLocalPath string, files []string, remotePath string, ...) error
- func (bd *BackupDestination) UploadPath(ctx context.Context, baseLocalPath string, files []string, remotePath string, ...) (int64, error)
- type COS
- func (c *COS) Close(ctx context.Context) error
- func (c *COS) Connect(ctx context.Context) error
- func (c *COS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
- func (c *COS) DeleteFile(ctx context.Context, key string) error
- func (c *COS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
- func (c *COS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
- func (c *COS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
- func (c *COS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)
- func (c *COS) Kind() string
- func (c *COS) PutFile(ctx context.Context, key string, r io.ReadCloser) error
- func (c *COS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error
- func (c *COS) StatFile(ctx context.Context, key string) (RemoteFile, error)
- func (c *COS) Walk(ctx context.Context, cosPath string, recursive bool, ...) error
- func (c *COS) WalkAbsolute(ctx context.Context, prefix string, recursive bool, ...) error
- type FTP
- func (f *FTP) Close(ctx context.Context) error
- func (f *FTP) Connect(ctx context.Context) error
- func (f *FTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
- func (f *FTP) DeleteFile(ctx context.Context, key string) error
- func (f *FTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
- func (f *FTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
- func (f *FTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
- func (f *FTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)
- func (f *FTP) Kind() string
- func (f *FTP) MkdirAll(key string, client *ftp.ServerConn) error
- func (f *FTP) PutFile(ctx context.Context, key string, r io.ReadCloser) error
- func (f *FTP) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error
- func (f *FTP) StatFile(ctx context.Context, key string) (RemoteFile, error)
- func (f *FTP) Walk(ctx context.Context, ftpPath string, recursive bool, ...) error
- func (f *FTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, ...) error
- type FTPFileReader
- type GCS
- func (gcs *GCS) Close(ctx context.Context) error
- func (gcs *GCS) Connect(ctx context.Context) error
- func (gcs *GCS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
- func (gcs *GCS) DeleteFile(ctx context.Context, key string) error
- func (gcs *GCS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
- func (gcs *GCS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
- func (gcs *GCS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
- func (gcs *GCS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)
- func (gcs *GCS) Kind() string
- func (gcs *GCS) PutFile(ctx context.Context, key string, r io.ReadCloser) error
- func (gcs *GCS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error
- func (gcs *GCS) StatFile(ctx context.Context, key string) (RemoteFile, error)
- func (gcs *GCS) Walk(ctx context.Context, gcsPath string, recursive bool, ...) error
- func (gcs *GCS) WalkAbsolute(ctx context.Context, rootPath string, recursive bool, ...) error
- type RecalculateV4Signature
- type RemoteFile
- type RemoteStorage
- type S3
- func (s *S3) Close(ctx context.Context) error
- func (s *S3) Connect(ctx context.Context) error
- func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
- func (s *S3) DeleteFile(ctx context.Context, key string) error
- func (s *S3) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
- func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
- func (s *S3) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
- func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error)
- func (s *S3) Kind() string
- func (s *S3) PutFile(ctx context.Context, key string, r io.ReadCloser) error
- func (s *S3) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error
- func (s *S3) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (endpoint smithyendpoints.Endpoint, err error)
- func (s *S3) StatFile(ctx context.Context, key string) (RemoteFile, error)
- func (s *S3) Walk(ctx context.Context, s3Path string, recursive bool, ...) error
- func (s *S3) WalkAbsolute(ctx context.Context, prefix string, recursive bool, ...) error
- type S3LogToZeroLogAdapter
- type SFTP
- func (sftp *SFTP) Close(ctx context.Context) error
- func (sftp *SFTP) Connect(ctx context.Context) error
- func (sftp *SFTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
- func (sftp *SFTP) Debug(msg string, v ...interface{})
- func (sftp *SFTP) DeleteDirectory(ctx context.Context, dirPath string) error
- func (sftp *SFTP) DeleteFile(ctx context.Context, key string) error
- func (sftp *SFTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
- func (sftp *SFTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
- func (sftp *SFTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
- func (sftp *SFTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error)
- func (sftp *SFTP) Kind() string
- func (sftp *SFTP) PutFile(ctx context.Context, key string, localFile io.ReadCloser) error
- func (sftp *SFTP) PutFileAbsolute(ctx context.Context, key string, localFile io.ReadCloser) error
- func (sftp *SFTP) StatFile(ctx context.Context, key string) (RemoteFile, error)
- func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, ...) error
- func (sftp *SFTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, ...) error
Constants ¶
View Source
const (
// BufferSize - size of ring buffer between stream handlers
BufferSize = 128 * 1024
)
Variables ¶
View Source
var ( // ErrNotFound is returned when file/object cannot be found ErrNotFound = errors.New("key not found") )
Functions ¶
func AdjustAzblobBufferSize ¶ added in v2.6.0
func AdjustS3PartSize ¶ added in v2.6.0
Types ¶
type AzureBlob ¶
type AzureBlob struct { Container azblob.ContainerURL Pipeline pipeline.Pipeline CPK azblob.ClientProvidedKeyOptions Config *config.AzureBlobConfig }
AzureBlob - presents methods for manipulate data on Azure
func (*AzureBlob) CopyObject ¶
func (*AzureBlob) DeleteFile ¶
func (*AzureBlob) DeleteFileFromObjectDiskBackup ¶
func (*AzureBlob) GetFileReader ¶
func (*AzureBlob) GetFileReaderAbsolute ¶ added in v2.5.0
func (*AzureBlob) GetFileReaderWithLocalPath ¶
func (*AzureBlob) PutFileAbsolute ¶ added in v2.5.0
type Backup ¶
type Backup struct { metadata.BackupMetadata Broken string UploadDate time.Time `json:"upload_date"` }
func GetBackupsToDeleteRemote ¶ added in v2.4.28
type BackupDestination ¶
type BackupDestination struct { RemoteStorage // contains filtered or unexported fields }
func NewBackupDestination ¶
func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhouse.ClickHouse, backupName string) (*BackupDestination, error)
func (*BackupDestination) BackupList ¶
func (*BackupDestination) DownloadCompressedStream ¶
func (*BackupDestination) DownloadPath ¶
func (*BackupDestination) RemoveBackupRemote ¶ added in v2.4.28
func (*BackupDestination) UploadCompressedStream ¶
type COS ¶
func (*COS) CopyObject ¶
func (*COS) DeleteFileFromObjectDiskBackup ¶
func (*COS) GetFileReader ¶
func (*COS) GetFileReaderAbsolute ¶ added in v2.5.0
func (*COS) GetFileReaderWithLocalPath ¶
func (*COS) PutFileAbsolute ¶ added in v2.5.0
type FTP ¶
func (*FTP) CopyObject ¶
func (*FTP) DeleteFileFromObjectDiskBackup ¶
func (*FTP) GetFileReader ¶
func (*FTP) GetFileReaderAbsolute ¶ added in v2.5.0
func (*FTP) GetFileReaderWithLocalPath ¶
func (*FTP) PutFileAbsolute ¶ added in v2.5.0
type FTPFileReader ¶
func (*FTPFileReader) Close ¶
func (fr *FTPFileReader) Close() error
type GCS ¶
GCS - presents methods for manipulate data on GCS
func (*GCS) CopyObject ¶
func (*GCS) DeleteFileFromObjectDiskBackup ¶
func (*GCS) GetFileReader ¶
func (*GCS) GetFileReaderAbsolute ¶ added in v2.5.0
func (*GCS) GetFileReaderWithLocalPath ¶
func (*GCS) PutFileAbsolute ¶ added in v2.5.0
type RecalculateV4Signature ¶
type RecalculateV4Signature struct {
// contains filtered or unexported fields
}
RecalculateV4Signature allow GCS over S3, remove Accept-Encoding header from sign https://stackoverflow.com/a/74382598/1204665, https://github.com/aws/aws-sdk-go-v2/issues/1816
type RemoteFile ¶
RemoteFile - interface describe file on remote storage
type RemoteStorage ¶
type RemoteStorage interface { Kind() string Connect(ctx context.Context) error Close(ctx context.Context) error StatFile(ctx context.Context, key string) (RemoteFile, error) DeleteFile(ctx context.Context, key string) error DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error Walk(ctx context.Context, prefix string, recursive bool, fn func(context.Context, RemoteFile) error) error WalkAbsolute(ctx context.Context, absolutePrefix string, recursive bool, fn func(context.Context, RemoteFile) error) error GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error) PutFile(ctx context.Context, key string, r io.ReadCloser) error PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) }
RemoteStorage -
type S3 ¶
type S3 struct { Config *config.S3Config PartSize int64 Concurrency int BufferSize int // contains filtered or unexported fields }
S3 - presents methods for manipulate data on s3
func (*S3) CopyObject ¶
func (*S3) DeleteFileFromObjectDiskBackup ¶
func (*S3) GetFileReader ¶
func (*S3) GetFileReaderAbsolute ¶ added in v2.5.0
func (*S3) GetFileReaderWithLocalPath ¶
func (*S3) PutFileAbsolute ¶ added in v2.5.0
func (*S3) ResolveEndpoint ¶ added in v2.6.0
func (s *S3) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (endpoint smithyendpoints.Endpoint, err error)
type S3LogToZeroLogAdapter ¶ added in v2.5.22
type S3LogToZeroLogAdapter struct {
// contains filtered or unexported fields
}
func (S3LogToZeroLogAdapter) Logf ¶ added in v2.5.22
func (S3LogToApexLogAdapter S3LogToZeroLogAdapter) Logf(severity awsV2Logging.Classification, msg string, args ...interface{})
type SFTP ¶
type SFTP struct { Config *config.SFTPConfig // contains filtered or unexported fields }
SFTP Implement RemoteStorage
func (*SFTP) CopyObject ¶
func (*SFTP) DeleteDirectory ¶
func (*SFTP) DeleteFileFromObjectDiskBackup ¶
func (*SFTP) GetFileReader ¶
func (*SFTP) GetFileReaderAbsolute ¶ added in v2.5.0
func (*SFTP) GetFileReaderWithLocalPath ¶
func (*SFTP) PutFileAbsolute ¶ added in v2.5.0
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK
|
Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK |
Click to show internal directories.
Click to hide internal directories.