object_disk

Documentation

Index

• Variables
• func ConvertLocalPathToRemote(diskName, localPath string) (string, error)
• func CopyObject(ctx context.Context, diskName string, srcSize int64, ...) (int64, error)
• func DeleteFile(ctx context.Context, diskName, remotePath string) error
• func GetFileReader(ctx context.Context, diskName, remotePath string) (io.ReadCloser, error)
• func InitCredentialsAndConnections(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
• func PutFile(ctx context.Context, diskName, remotePath string, content []byte) error
• func ReadBoolText(scanner *bufio.Scanner) (bool, error)
• func ReadFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) ([]byte, error)
• func ReadInt64Text(scanner *bufio.Scanner) (int64, error)
• func ReadIntText(scanner *bufio.Scanner) (int, error)
• func WriteFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, ...) error
• func WriteMetadataToFile(metadata *Metadata, path string) error
• type Metadata
  • func ReadMetadataFromFile(path string) (*Metadata, error)
  • func ReadMetadataFromReader(metadataFile io.ReadCloser, path string) (*Metadata, error)
• type MetadataVersion
• type Object
• type ObjectStorageConnection
  • func (c *ObjectStorageConnection) GetRemoteBucket() string
  • func (c *ObjectStorageConnection) GetRemotePath() string
  • func (c *ObjectStorageConnection) GetRemoteStorage() storage.RemoteStorage
• type ObjectStorageCredentials
• type StorageObject

Variables

var DisksConnections = xsync.NewMapOf[*ObjectStorageConnection]()

var DisksCredentials = xsync.NewMapOf[ObjectStorageCredentials]()

var InitCredentialsAndConnectionsMutex sync.Mutex

var SystemDisks = xsync.NewMapOf[clickhouse.Disk]()

Functions

func ConvertLocalPathToRemote

func ConvertLocalPathToRemote(diskName, localPath string) (string, error)

func CopyObject

func CopyObject(ctx context.Context, diskName string, srcSize int64, srcBucket, srcKey, dstPath string) (int64, error)

func DeleteFile

func DeleteFile(ctx context.Context, diskName, remotePath string) error

func GetFileReader

func GetFileReader(ctx context.Context, diskName, remotePath string) (io.ReadCloser, error)

func InitCredentialsAndConnections

func InitCredentialsAndConnections(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName string) error

func PutFile

func PutFile(ctx context.Context, diskName, remotePath string, content []byte) error

func ReadBoolText

func ReadBoolText(scanner *bufio.Scanner) (bool, error)

func ReadFileContent

func ReadFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string) ([]byte, error)

func ReadInt64Text

func ReadInt64Text(scanner *bufio.Scanner) (int64, error)

func ReadIntText

func ReadIntText(scanner *bufio.Scanner) (int, error)

func WriteFileContent

func WriteFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string, content []byte) error

func WriteMetadataToFile

func WriteMetadataToFile(metadata *Metadata, path string) error

Types

type Metadata

type Metadata struct {
	Version            MetadataVersion
	StorageObjectCount int
	TotalSize          int64
	StorageObjects     []StorageObject
	RefCount           int
	ReadOnly           bool
	InlineData         string
	Path               string
}

func ReadMetadataFromFile

func ReadMetadataFromFile(path string) (*Metadata, error)

func ReadMetadataFromReader

func ReadMetadataFromReader(metadataFile io.ReadCloser, path string) (*Metadata, error)

type MetadataVersion

type MetadataVersion uint32

const (
	VersionAbsolutePaths MetadataVersion = 1
	VersionRelativePath  MetadataVersion = 2
	VersionReadOnlyFlag  MetadataVersion = 3
	VersionInlineData    MetadataVersion = 4
)

origin https://github.com/ClickHouse/ClickHouse/blame/master/src/Disks/ObjectStorages/DiskObjectStorageMetadata.h#L15

type Object

type Object struct {
	Key      string
	Created  time.Time
	Deleted  time.Time
	Versions []string
	Deletes  []string
	Size     int64
}

type ObjectStorageConnection

type ObjectStorageConnection struct {
	Type         string
	S3           *storage.S3
	AzureBlob    *storage.AzureBlob
	MetadataPath string
}

func (*ObjectStorageConnection) GetRemoteBucket

func (c *ObjectStorageConnection) GetRemoteBucket() string

func (*ObjectStorageConnection) GetRemotePath

func (c *ObjectStorageConnection) GetRemotePath() string

func (*ObjectStorageConnection) GetRemoteStorage

func (c *ObjectStorageConnection) GetRemoteStorage() storage.RemoteStorage

type ObjectStorageCredentials

type ObjectStorageCredentials struct {
	Type               string // aws, gcs, azblob
	EndPoint           string
	S3AccessKey        string
	S3SecretKey        string
	S3AssumeRole       string
	S3Region           string
	S3StorageClass     string
	AzureAccountName   string
	AzureAccountKey    string
	AzureContainerName string
}

type StorageObject

type StorageObject struct {
	ObjectSize         int64
	ObjectRelativePath string
}