object_disk

Documentation

Index

• Variables
• func ConvertLocalPathToRemote(diskName, localPath string) (string, error)
• func CopyObject(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
• func DeleteFile(ctx context.Context, diskName, remotePath string) error
• func DeleteFileWithContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
• func GetFileReader(ctx context.Context, diskName, remotePath string) (io.ReadCloser, error)
• func GetFileSize(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) (int64, error)
• func InitCredentialsAndConnections(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
• func PutFile(ctx context.Context, diskName, remotePath string, content []byte) error
• func ReadBoolText(scanner *bufio.Scanner) (bool, error)
• func ReadFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) ([]byte, error)
• func ReadInt64Text(scanner *bufio.Scanner) (int64, error)
• func ReadIntText(scanner *bufio.Scanner) (int, error)
• func WriteFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
• func WriteMetadataToFile(metadata *Metadata, path string) error
• type Metadata
  • func ReadMetadataFromFile(path string) (*Metadata, error)
  • func ReadMetadataFromReader(metadataFile io.ReadCloser, path string) (*Metadata, error)
• type MetadataVersion
• type Object
• type ObjectStorageConnection
  • func (c *ObjectStorageConnection) GetRemoteBucket() string
  • func (c *ObjectStorageConnection) GetRemotePath() string
  • func (c *ObjectStorageConnection) GetRemoteStorage() storage.RemoteStorage
• type ObjectStorageCredentials
• type StorageObject

Variables

var DisksConnections map[string]ObjectStorageConnection

var DisksCredentials map[string]ObjectStorageCredentials

var SystemDisks map[string]clickhouse.Disk

Functions

func ConvertLocalPathToRemote

func ConvertLocalPathToRemote(diskName, localPath string) (string, error)

func CopyObject

func CopyObject(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, srcBucket, srcKey, dstPath string) error

func DeleteFile

func DeleteFile(ctx context.Context, diskName, remotePath string) error

func DeleteFileWithContent

func DeleteFileWithContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string) error

func GetFileReader

func GetFileReader(ctx context.Context, diskName, remotePath string) (io.ReadCloser, error)

func GetFileSize

func GetFileSize(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, remotePath string) (int64, error)

func InitCredentialsAndConnections

func InitCredentialsAndConnections(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName string) error

func PutFile

func PutFile(ctx context.Context, diskName, remotePath string, content []byte) error

func ReadBoolText

func ReadBoolText(scanner *bufio.Scanner) (bool, error)

func ReadFileContent

func ReadFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string) ([]byte, error)

func ReadInt64Text

func ReadInt64Text(scanner *bufio.Scanner) (int64, error)

func ReadIntText

func ReadIntText(scanner *bufio.Scanner) (int, error)

func WriteFileContent

func WriteFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string, content []byte) error

func WriteMetadataToFile

func WriteMetadataToFile(metadata *Metadata, path string) error

Types

type Metadata

type Metadata struct {
	Version            MetadataVersion
	StorageObjectCount int
	TotalSize          int64
	StorageObjects     []StorageObject
	RefCount           int
	ReadOnly           bool
	InlineData         string
	Path               string
}

func ReadMetadataFromFile

func ReadMetadataFromFile(path string) (*Metadata, error)

func ReadMetadataFromReader

func ReadMetadataFromReader(metadataFile io.ReadCloser, path string) (*Metadata, error)

type MetadataVersion

type MetadataVersion uint32

const (
	VersionAbsolutePaths MetadataVersion = 1
	VersionRelativePath  MetadataVersion = 2
	VersionReadOnlyFlag  MetadataVersion = 3
	VersionInlineData    MetadataVersion = 4
)

origin https://github.com/ClickHouse/ClickHouse/blame/master/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h#L15

type Object

type Object struct {
	Key      string
	Created  time.Time
	Deleted  time.Time
	Versions []string
	Deletes  []string
	Size     int64
}

type ObjectStorageConnection

type ObjectStorageConnection struct {
	Type         string
	S3           *storage.S3
	AzureBlob    *storage.AzureBlob
	MetadataPath string
}

func (*ObjectStorageConnection) GetRemoteBucket

func (c *ObjectStorageConnection) GetRemoteBucket() string

func (*ObjectStorageConnection) GetRemotePath

func (c *ObjectStorageConnection) GetRemotePath() string

func (*ObjectStorageConnection) GetRemoteStorage

func (c *ObjectStorageConnection) GetRemoteStorage() storage.RemoteStorage

type ObjectStorageCredentials

type ObjectStorageCredentials struct {
	Type               string // aws, gcs, azblob
	EndPoint           string
	S3AccessKey        string
	S3SecretKey        string
	S3AssumeRole       string
	S3Region           string
	S3StorageClass     string
	AzureAccountName   string
	AzureAccountKey    string
	AzureContainerName string
}

type StorageObject

type StorageObject struct {
	ObjectSize         int64
	ObjectRelativePath string
}