chbackup

package
v0.5.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 7, 2020 License: MIT Imports: 40 Imported by: 0

Documentation

Overview

Package chbackup describes the objects and functions for clickhouse-backup app

Index

Constants

View Source 
const (
	// MetaFileName - meta file name
	MetaFileName = "meta.json"
	// BufferSize - size of ring buffer between stream handlers
	BufferSize = 4 * 1024 * 1024
)
View Source 
const (
	// BackupTimeFormat - default backup name format
	BackupTimeFormat = "2006-01-02T15-04-05"
)

Variables

View Source 
var (
	// ErrNotFound is returned when file/object cannot be found
	ErrNotFound = errors.New("file not found")
)
View Source 
var (
	// ErrUnknownClickhouseDataPath -
	ErrUnknownClickhouseDataPath = errors.New("clickhouse data path is unknown, you can set data_path in config file")
)

Functions

func Clean 

func Clean(config Config) error

Clean - removed all data in shadow folder

func CreateBackup 

func CreateBackup(config Config, backupName, tablePattern string) error

CreateBackup - create new backup of all tables matched by tablePattern If backupName is empty string will use default backup name

func Download 

func Download(config Config, backupName string) error

func FormatBytes 

func FormatBytes(i int64) (result string)

FormatBytes - Convert bytes to human readable string

func Freeze 

func Freeze(config Config, tablePattern string) error

Freeze - freeze tables by tablePattern

func GetLocalBackup 

func GetLocalBackup(config Config, backupName string) error

func NewBackupName 

func NewBackupName() string

NewBackupName - return default backup name

func PrintDefaultConfig 

func PrintDefaultConfig()

PrintDefaultConfig - print default config to stdout

func PrintLocalBackups 

func PrintLocalBackups(config Config, format string) error

PrintLocalBackups - print all backups stored locally

func PrintRemoteBackups 

func PrintRemoteBackups(config Config, format string) error

PrintRemoteBackups - print all backups stored on remote storage

func PrintTables 

func PrintTables(config Config) error

PrintTables - print all tables suitable for backup

func RemoveBackupLocal 

func RemoveBackupLocal(config Config, backupName string) error

func RemoveBackupRemote 

func RemoveBackupRemote(config Config, backupName string) error

func RemoveOldBackupsLocal 

func RemoveOldBackupsLocal(config Config) error

func Restore 

func Restore(config Config, backupName string, tablePattern string, schemaOnly bool, dataOnly bool) error

Restore - restore tables matched by tablePattern from backupName

func RestoreData 

func RestoreData(config Config, backupName string, tablePattern string) error

RestoreData - restore data for tables matched by tablePattern from backupName

func TablePathEncode 

func TablePathEncode(str string) string

func Upload 

func Upload(config Config, backupName string, diffFrom string) error

Types

type Backup 

type Backup struct {
	Name string
	Size int64
	Date time.Time
}

func GetBackupsToDelete 

func GetBackupsToDelete(backups []Backup, keep int) []Backup

func ListLocalBackups 

func ListLocalBackups(config Config) ([]Backup, error)

ListLocalBackups - return slice of all backups stored locally

type BackupDestination 

type BackupDestination struct {
	RemoteStorage
	// contains filtered or unexported fields
}

func NewBackupDestination 

func NewBackupDestination(config Config) (*BackupDestination, error)

func (*BackupDestination) BackupList 

func (bd *BackupDestination) BackupList() ([]Backup, error)

func (*BackupDestination) BackupsToKeep 

func (bd *BackupDestination) BackupsToKeep() int

func (*BackupDestination) CompressedStreamDownload 

func (bd *BackupDestination) CompressedStreamDownload(remotePath string, localPath string) error

func (*BackupDestination) CompressedStreamUpload 

func (bd *BackupDestination) CompressedStreamUpload(localPath, remotePath, diffFromPath string) error

func (*BackupDestination) RemoveBackup 

func (bd *BackupDestination) RemoveBackup(backupName string) error

func (*BackupDestination) RemoveOldBackups 

func (bd *BackupDestination) RemoveOldBackups(keep int) error

type BackupPartition 

type BackupPartition struct {
	Name string
	Path string
}

BackupPartition - struct representing Clickhouse partition

type BackupTable 

type BackupTable struct {
	Database   string
	Name       string
	Partitions []BackupPartition
}

BackupTable - struct to store additional information on partitions

type BackupTables 

type BackupTables []BackupTable

BackupTables - slice of BackupTable

func (BackupTables) Sort 

func (bt BackupTables) Sort()

Sort - sorting BackupTables slice orderly by name

type Bar 

type Bar struct {
	// contains filtered or unexported fields
}

func StartNewBar 

func StartNewBar(show bool, total int) *Bar

func StartNewByteBar 

func StartNewByteBar(show bool, total int64) *Bar

func (*Bar) Add64 

func (b *Bar) Add64(add int64)

func (*Bar) Finish 

func (b *Bar) Finish()

func (*Bar) Increment 

func (b *Bar) Increment()

func (*Bar) NewProxyReader 

func (b *Bar) NewProxyReader(r io.Reader) io.Reader

func (*Bar) Set 

func (b *Bar) Set(current int)

type COS 

type COS struct {
	Config *COSConfig
	// contains filtered or unexported fields
}

func (*COS) Connect 

func (c *COS) Connect() error

Connect - connect to cos

func (*COS) DeleteFile 

func (c *COS) DeleteFile(key string) error

func (*COS) GetFile 

func (c *COS) GetFile(key string) (RemoteFile, error)

func (*COS) GetFileReader 

func (c *COS) GetFileReader(key string) (io.ReadCloser, error)

func (*COS) Kind 

func (c *COS) Kind() string

func (*COS) PutFile 

func (c *COS) PutFile(key string, r io.ReadCloser) error

func (*COS) Walk 

func (c *COS) Walk(path string, process func(RemoteFile)) error

type COSConfig 

type COSConfig struct {
	RowURL            string `yaml:"url" envconfig:"COS_URL"`
	Timeout           string `yaml:"timeout" envconfig:"COS_TIMEOUT"`
	SecretID          string `yaml:"secret_id" envconfig:"COS_SECRET_ID"`
	SecretKey         string `yaml:"secret_key" envconfig:"COS_SECRET_KEY"`
	Path              string `yaml:"path" envconfig:"COS_PATH"`
	CompressionFormat string `yaml:"compression_format" envconfig:"COS_COMPRESSION_FORMAT"`
	CompressionLevel  int    `yaml:"compression_level" envconfig:"COS_COMPRESSION_LEVEL"`
	Debug             bool   `yaml:"debug" envconfig:"COS_DEBUG"`
}

COSConfig - cos settings section

type ClickHouse 

type ClickHouse struct {
	Config *ClickHouseConfig
	// contains filtered or unexported fields
}

ClickHouse - provide

func (*ClickHouse) AttachPatritions 

func (ch *ClickHouse) AttachPatritions(table BackupTable) error

AttachPatritions - execute ATTACH command for specific table

func (*ClickHouse) Chown 

func (ch *ClickHouse) Chown(filename string) error

Chown - set permission on file to clickhouse user This is necessary that the ClickHouse will be able to read parts files on restore

func (*ClickHouse) Close 

func (ch *ClickHouse) Close() error

Close - closing connection to ClickHouse

func (*ClickHouse) Connect 

func (ch *ClickHouse) Connect() error

Connect - establish connection to ClickHouse

func (*ClickHouse) CopyData 

func (ch *ClickHouse) CopyData(table BackupTable) error

CopyData - copy partitions for specific table to detached folder

func (*ClickHouse) CreateDatabase 

func (ch *ClickHouse) CreateDatabase(database string) error

CreateDatabase - create ClickHouse database

func (*ClickHouse) CreateTable 

func (ch *ClickHouse) CreateTable(table RestoreTable) error

CreateTable - create ClickHouse table

func (*ClickHouse) FreezeTable 

func (ch *ClickHouse) FreezeTable(table Table) error

FreezeTable - freeze all partitions for table This way available for ClickHouse sience v19.1

func (*ClickHouse) FreezeTableOldWay 

func (ch *ClickHouse) FreezeTableOldWay(table Table) error

FreezeTableOldWay - freeze all partitions in table one by one This way using for ClickHouse below v19.1

func (*ClickHouse) GetBackupTables 

func (ch *ClickHouse) GetBackupTables(backupName string) (map[string]BackupTable, error)

GetBackupTables - return list of backups of tables that can be restored

func (*ClickHouse) GetConn 

func (ch *ClickHouse) GetConn() *sqlx.DB

GetConn - return current connection

func (*ClickHouse) GetDataPath 

func (ch *ClickHouse) GetDataPath() (string, error)

GetDataPath - return ClickHouse data_path

func (*ClickHouse) GetTables 

func (ch *ClickHouse) GetTables() ([]Table, error)

GetTables - return slice of all tables suitable for backup

func (*ClickHouse) GetVersion 

func (ch *ClickHouse) GetVersion() (int, error)

GetVersion - returned ClickHouse version in number format Example value: 19001005

type ClickHouseConfig 

type ClickHouseConfig struct {
	Username     string   `yaml:"username" envconfig:"CLICKHOUSE_USERNAME"`
	Password     string   `yaml:"password" envconfig:"CLICKHOUSE_PASSWORD"`
	Host         string   `yaml:"host" envconfig:"CLICKHOUSE_HOST"`
	Port         uint     `yaml:"port" envconfig:"CLICKHOUSE_PORT"`
	DataPath     string   `yaml:"data_path" envconfig:"CLICKHOUSE_DATA_PATH"`
	SkipTables   []string `yaml:"skip_tables" envconfig:"CLICKHOUSE_SKIP_TABLES"`
	Timeout      string   `yaml:"timeout" envconfig:"CLICKHOUSE_TIMEOUT"`
	FreezeByPart bool     `yaml:"freeze_by_part" envconfig:"CLICKHOUSE_FREEZE_BY_PART"`
}

ClickHouseConfig - clickhouse settings section

type Config 

type Config struct {
	General    GeneralConfig    `yaml:"general"`
	ClickHouse ClickHouseConfig `yaml:"clickhouse"`
	S3         S3Config         `yaml:"s3"`
	GCS        GCSConfig        `yaml:"gcs"`
	COS        COSConfig        `yaml:"cos"`
}

Config - config file format

func DefaultConfig 

func DefaultConfig() *Config

func LoadConfig 

func LoadConfig(configLocation string) (*Config, error)

LoadConfig - load config from file

type GCS 

type GCS struct {
	Config *GCSConfig
	// contains filtered or unexported fields
}

GCS - presents methods for manipulate data on GCS

func (*GCS) Connect 

func (gcs *GCS) Connect() error

Connect - connect to GCS

func (*GCS) DeleteFile 

func (gcs *GCS) DeleteFile(key string) error

func (*GCS) GetFile 

func (gcs *GCS) GetFile(key string) (RemoteFile, error)

func (*GCS) GetFileReader 

func (gcs *GCS) GetFileReader(key string) (io.ReadCloser, error)

func (*GCS) GetFileWriter 

func (gcs *GCS) GetFileWriter(key string) io.WriteCloser

func (*GCS) Kind 

func (gcs *GCS) Kind() string

func (*GCS) PutFile 

func (gcs *GCS) PutFile(key string, r io.ReadCloser) error

func (*GCS) Walk 

func (gcs *GCS) Walk(gcsPath string, process func(r RemoteFile)) error

type GCSConfig 

type GCSConfig struct {
	CredentialsFile   string `yaml:"credentials_file" envconfig:"GCS_CREDENTIALS_FILE"`
	CredentialsJSON   string `yaml:"credentials_json" envconfig:"GCS_CREDENTIALS_JSON"`
	Bucket            string `yaml:"bucket" envconfig:"GCS_BUCKET"`
	Path              string `yaml:"path" envconfig:"GCS_PATH"`
	CompressionLevel  int    `yaml:"compression_level" envconfig:"GCS_COMPRESSION_LEVEL"`
	CompressionFormat string `yaml:"compression_format" envconfig:"GCS_COMPRESSION_FORMAT"`
}

GCSConfig - GCS settings section

type GeneralConfig 

type GeneralConfig struct {
	RemoteStorage       string `yaml:"remote_storage" envconfig:"REMOTE_STORAGE"`
	DisableProgressBar  bool   `yaml:"disable_progress_bar" envconfig:"DISABLE_PROGRESS_BAR"`
	BackupsToKeepLocal  int    `yaml:"backups_to_keep_local" envconfig:"BACKUPS_TO_KEEP_LOCAL"`
	BackupsToKeepRemote int    `yaml:"backups_to_keep_remote" envconfig:"BACKUPS_TO_KEEP_REMOTE"`
}

GeneralConfig - general setting section

type MetaFile 

type MetaFile struct {
	RequiredBackup string   `json:"required_backup"`
	Hardlinks      []string `json:"hardlinks"`
}

MetaFile - structure describe meta file that will be added to incremental backups archive. Contains info of required files in backup and files

type RemoteFile 

type RemoteFile interface {
	Size() int64
	Name() string
	LastModified() time.Time
}

RemoteFile - interface describe file on remote storage

type RemoteStorage 

type RemoteStorage interface {
	Kind() string
	GetFile(string) (RemoteFile, error)
	DeleteFile(string) error
	Connect() error
	Walk(string, func(RemoteFile)) error
	GetFileReader(key string) (io.ReadCloser, error)
	PutFile(key string, r io.ReadCloser) error
}

RemoteStorage -

type RestoreTable 

type RestoreTable struct {
	Database string
	Table    string
	Query    string
	Path     string
}

RestoreTable - struct to store information needed during restore

type RestoreTables 

type RestoreTables []RestoreTable

RestoreTables - slice of RestoreTable

func (RestoreTables) Sort 

func (rt RestoreTables) Sort()

Sort - sorting BackupTables slice orderly by name

type S3 

type S3 struct {
	Config *S3Config
	// contains filtered or unexported fields
}

S3 - presents methods for manipulate data on s3

func (*S3) Connect 

func (s *S3) Connect() error

Connect - connect to s3

func (*S3) DeleteFile 

func (s *S3) DeleteFile(key string) error

func (*S3) GetFile 

func (s *S3) GetFile(key string) (RemoteFile, error)

func (*S3) GetFileReader 

func (s *S3) GetFileReader(key string) (io.ReadCloser, error)

func (*S3) Kind 

func (s *S3) Kind() string

func (*S3) PutFile 

func (s *S3) PutFile(key string, r io.ReadCloser) error

func (*S3) Walk 

func (s *S3) Walk(s3Path string, process func(r RemoteFile)) error

type S3Config 

type S3Config struct {
	AccessKey               string `yaml:"access_key" envconfig:"S3_ACCESS_KEY"`
	SecretKey               string `yaml:"secret_key" envconfig:"S3_SECRET_KEY"`
	Bucket                  string `yaml:"bucket" envconfig:"S3_BUCKET"`
	Endpoint                string `yaml:"endpoint" envconfig:"S3_ENDPOINT"`
	Region                  string `yaml:"region" envconfig:"S3_REGION"`
	ACL                     string `yaml:"acl" envconfig:"S3_ACL"`
	ForcePathStyle          bool   `yaml:"force_path_style" envconfig:"S3_FORCE_PATH_STYLE"`
	Path                    string `yaml:"path" envconfig:"S3_PATH"`
	DisableSSL              bool   `yaml:"disable_ssl" envconfig:"S3_DISABLE_SSL"`
	PartSize                int64  `yaml:"part_size" envconfig:"S3_PART_SIZE"`
	CompressionLevel        int    `yaml:"compression_level" envconfig:"S3_COMPRESSION_LEVEL"`
	CompressionFormat       string `yaml:"compression_format" envconfig:"S3_COMPRESSION_FORMAT"`
	SSE                     string `yaml:"sse" envconfig:"S3_SSE"`
	DisableCertVerification bool   `yaml:"disable_cert_verification" envconfig:"S3_DISABLE_CERT_VERIFICATION"`
	MaxRetry                int    `yaml:"max_retry" envconfig:"S3_MAX_RETRY"`
}

S3Config - s3 settings section

type Table 

type Table struct {
	Database string `db:"database"`
	Name     string `db:"name"`
	Skip     bool
}

Table - ClickHouse table struct

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.