storage

package
v0.4.15 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 13, 2024 License: Apache-2.0 Imports: 33 Imported by: 0

Documentation

Index

Constants

View Source 
const NoSuchKey = "NoSuchKey"

Variables

View Source 
var CheckBucketRetryAttempts uint = 20
View Source 
var (
	ErrNoSuchKey = errors.New("NoSuchKey")
)

Functions

func CopyDir 

func CopyDir(source string, dest string) (err error)

func CopyFile 

func CopyFile(source string, dest string) (err error)

func IsErrNoSuchKey 

func IsErrNoSuchKey(err error) bool

func Read 

func Read(r io.Reader, size int64) ([]byte, error)

Learn from file.ReadFile

func ReadFile 

func ReadFile(filePath string) ([]byte, error)

ReadFile reads file as os.ReadFile works, also converts the os errors to Milvus errors

func WrapErrFileNotFound 

func WrapErrFileNotFound(key string) error

func WrapErrNoSuchKey 

func WrapErrNoSuchKey(key string) error

func WriteFile 

func WriteFile(filePath string, data []byte, perm fs.FileMode) error

WriteFile writes file as os.WriteFile works， also converts the os errors to Milvus errors

Types

type AzureChunkManager 

type AzureChunkManager struct {
	// contains filtered or unexported fields
}

AzureChunkManager is responsible for read and write data stored in minio.

func NewAzureChunkManager 

func NewAzureChunkManager(ctx context.Context, c *config) (*AzureChunkManager, error)

func (*AzureChunkManager) Copy 

func (mcm *AzureChunkManager) Copy(ctx context.Context, fromBucketName string, toBucketName string, fromPath string, toPath string) error

func (*AzureChunkManager) Exist 

func (mcm *AzureChunkManager) Exist(ctx context.Context, bucketName string, filePath string) (bool, error)

Exist checks whether chunk is saved to minio storage.

func (*AzureChunkManager) ListWithPrefix 

func (mcm *AzureChunkManager) ListWithPrefix(ctx context.Context, bucketName string, prefix string, recursive bool) ([]string, []int64, error)

ListWithPrefix returns objects with provided prefix.

func (*AzureChunkManager) MultiRead 

func (mcm *AzureChunkManager) MultiRead(ctx context.Context, bucketName string, keys []string) ([][]byte, error)

func (*AzureChunkManager) MultiRemove 

func (mcm *AzureChunkManager) MultiRemove(ctx context.Context, bucketName string, keys []string) error

MultiRemove deletes a objects with @keys.

func (*AzureChunkManager) MultiWrite 

func (mcm *AzureChunkManager) MultiWrite(ctx context.Context, bucketName string, kvs map[string][]byte) error

MultiWrite saves multiple objects, the path is the key of @kvs. The object value is the value of @kvs.

func (*AzureChunkManager) Path 

func (mcm *AzureChunkManager) Path(ctx context.Context, bucketName string, filePath string) (string, error)

Path returns the path of minio data if exists.

func (*AzureChunkManager) Read 

func (mcm *AzureChunkManager) Read(ctx context.Context, bucketName string, filePath string) ([]byte, error)

Read reads the minio storage data if exists.

func (*AzureChunkManager) ReadWithPrefix 

func (mcm *AzureChunkManager) ReadWithPrefix(ctx context.Context, bucketName string, prefix string) ([]string, [][]byte, error)

func (*AzureChunkManager) Reader 

func (mcm *AzureChunkManager) Reader(ctx context.Context, bucketName string, filePath string) (FileReader, error)

Reader returns the path of minio data if exists.

func (*AzureChunkManager) Remove 

func (mcm *AzureChunkManager) Remove(ctx context.Context, bucketName string, filePath string) error

Remove deletes an object with @key.

func (*AzureChunkManager) RemoveWithPrefix 

func (mcm *AzureChunkManager) RemoveWithPrefix(ctx context.Context, bucketName string, prefix string) error

RemoveWithPrefix removes all objects with the same prefix @prefix from minio.

func (*AzureChunkManager) Size 

func (mcm *AzureChunkManager) Size(ctx context.Context, bucketName string, filePath string) (int64, error)

func (*AzureChunkManager) Write 

func (mcm *AzureChunkManager) Write(ctx context.Context, bucketName string, filePath string, content []byte) error

Write writes the data to minio storage.

type AzureObjectStorage 

type AzureObjectStorage struct {
	// contains filtered or unexported fields
}

func (*AzureObjectStorage) CopyObject 

func (aos *AzureObjectStorage) CopyObject(ctx context.Context, fromBucketName, toBucketName, fromPath, toPath string) error

func (*AzureObjectStorage) GetObject 

func (aos *AzureObjectStorage) GetObject(ctx context.Context, bucketName, objectName string, offset int64, size int64) (FileReader, error)

func (*AzureObjectStorage) ListObjects 

func (aos *AzureObjectStorage) ListObjects(ctx context.Context, bucketName string, prefix string, recursive bool) (map[string]int64, error)

func (*AzureObjectStorage) PutObject 

func (aos *AzureObjectStorage) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64) error

func (*AzureObjectStorage) RemoveObject 

func (aos *AzureObjectStorage) RemoveObject(ctx context.Context, bucketName, objectName string) error

func (*AzureObjectStorage) StatObject 

func (aos *AzureObjectStorage) StatObject(ctx context.Context, bucketName, objectName string) (int64, error)

type ChunkManager 

type ChunkManager interface {

	// Path returns path of @filePath.
	Path(ctx context.Context, bucketName string, filePath string) (string, error)
	// Size returns path of @filePath.
	Size(ctx context.Context, bucketName string, filePath string) (int64, error)
	// Write writes @content to @filePath.
	Write(ctx context.Context, bucketName string, filePath string, content []byte) error
	// MultiWrite writes multi @content to @filePath.
	//MultiWrite(ctx context.Context, bucketName string, contents map[string][]byte) error
	// Exist returns true if @filePath exists.
	Exist(ctx context.Context, bucketName string, filePath string) (bool, error)
	// Read reads @filePath and returns content.
	Read(ctx context.Context, bucketName string, filePath string) ([]byte, error)
	// Reader return a reader for @filePath
	//Reader(ctx context.Context, bucketName string, filePath string) (FileReader, error)
	// MultiRead reads @filePath and returns content.
	//MultiRead(ctx context.Context, bucketName string, filePaths []string) ([][]byte, error)
	ListWithPrefix(ctx context.Context, bucketName string, prefix string, recursive bool) ([]string, []int64, error)
	// ReadWithPrefix reads files with same @prefix and returns contents.
	//ReadWithPrefix(ctx context.Context, bucketName string, prefix string) ([]string, [][]byte, error)
	// Not use
	//Mmap(ctx context.Context, bucketName string, filePath string) (*mmap.ReaderAt, error)
	// ReadAt reads @filePath by offset @off, content stored in @p, return @n as the number of bytes read.
	// if all bytes are read, @err is io.EOF.
	// return other error if read failed.
	//ReadAt(ctx context.Context, bucketName string, filePath string, off int64, length int64) (p []byte, err error)
	// Remove delete @filePath.
	Remove(ctx context.Context, bucketName string, filePath string) error
	// MultiRemove delete @filePaths.
	//MultiRemove(ctx context.Context, bucketName string, filePaths []string) error
	// RemoveWithPrefix remove files with same @prefix.
	RemoveWithPrefix(ctx context.Context, bucketName string, prefix string) error
	// Move move files from fromPath into toPath recursively
	Copy(ctx context.Context, fromBucketName string, toBucketName string, fromPath string, toPath string) error
}

ChunkManager is to manager chunks. Include Read, Write, Remove chunks.

func NewChunkManager 

func NewChunkManager(ctx context.Context, params paramtable.BackupParams) (ChunkManager, error)

type FileReader 

type FileReader interface {
	io.Reader
	io.Closer
}

type LocalChunkManager 

type LocalChunkManager struct {
	// contains filtered or unexported fields
}

LocalChunkManager is responsible for read and write local file.

func NewLocalChunkManager 

func NewLocalChunkManager(ctx context.Context, c *config) (*LocalChunkManager, error)

NewLocalChunkManager create a new local manager object.

func (*LocalChunkManager) Copy 

func (lcm *LocalChunkManager) Copy(ctx context.Context, fromBucketName string, toBucketName string, fromPath string, toPath string) error

func (*LocalChunkManager) Exist 

func (lcm *LocalChunkManager) Exist(ctx context.Context, bucketName string, filePath string) (bool, error)

Exist checks whether chunk is saved to local storage.

func (*LocalChunkManager) ListWithPrefix 

func (lcm *LocalChunkManager) ListWithPrefix(ctx context.Context, bucketName string, prefix string, recursive bool) ([]string, []int64, error)

func (*LocalChunkManager) MultiRemove 

func (lcm *LocalChunkManager) MultiRemove(ctx context.Context, bucketName string, filePaths []string) error

func (*LocalChunkManager) Path 

func (lcm *LocalChunkManager) Path(ctx context.Context, bucketName string, filePath string) (string, error)

Path returns the path of local data if exists.

func (*LocalChunkManager) Read 

func (lcm *LocalChunkManager) Read(ctx context.Context, bucketName string, filePath string) ([]byte, error)

Read reads the local storage data if exists.

func (*LocalChunkManager) Remove 

func (lcm *LocalChunkManager) Remove(ctx context.Context, bucketName string, filePath string) error

func (*LocalChunkManager) RemoveWithPrefix 

func (lcm *LocalChunkManager) RemoveWithPrefix(ctx context.Context, bucketName string, prefix string) error

func (*LocalChunkManager) RootPath 

func (lcm *LocalChunkManager) RootPath() string

RootPath returns lcm root path.

func (*LocalChunkManager) Size 

func (lcm *LocalChunkManager) Size(ctx context.Context, bucketName string, filePath string) (int64, error)

func (*LocalChunkManager) Write 

func (lcm *LocalChunkManager) Write(ctx context.Context, bucketName string, filePath string, content []byte) error

Write writes the data to local storage.

type MinioChunkManager 

type MinioChunkManager struct {
	*minio.Client
	// contains filtered or unexported fields
}

MinioChunkManager is responsible for read and write data stored in minio.

func NewMinioChunkManager 

func NewMinioChunkManager(ctx context.Context, opts ...Option) (*MinioChunkManager, error)

NewMinioChunkManager create a new local manager object. Do not call this directly! Use factory.NewPersistentStorageChunkManager instead.

func (*MinioChunkManager) Copy 

func (mcm *MinioChunkManager) Copy(ctx context.Context, fromBucketName string, toBucketName string, fromPath string, toPath string) error

func (*MinioChunkManager) Exist 

func (mcm *MinioChunkManager) Exist(ctx context.Context, bucketName string, filePath string) (bool, error)

Exist checks whether chunk is saved to minio storage.

func (*MinioChunkManager) ListWithPrefix 

func (mcm *MinioChunkManager) ListWithPrefix(ctx context.Context, bucketName string, prefix string, recursive bool) ([]string, []int64, error)

func (*MinioChunkManager) MultiRead 

func (mcm *MinioChunkManager) MultiRead(ctx context.Context, bucketName string, keys []string) ([][]byte, error)

func (*MinioChunkManager) MultiRemove 

func (mcm *MinioChunkManager) MultiRemove(ctx context.Context, bucketName string, keys []string) error

MultiRemove deletes a objects with @keys.

func (*MinioChunkManager) MultiWrite 

func (mcm *MinioChunkManager) MultiWrite(ctx context.Context, bucketName string, kvs map[string][]byte) error

MultiWrite saves multiple objects, the path is the key of @kvs. The object value is the value of @kvs.

func (*MinioChunkManager) Path 

func (mcm *MinioChunkManager) Path(ctx context.Context, bucketName string, filePath string) (string, error)

Path returns the path of minio data if exists.

func (*MinioChunkManager) Read 

func (mcm *MinioChunkManager) Read(ctx context.Context, bucketName string, filePath string) ([]byte, error)

Read reads the minio storage data if exists.

func (*MinioChunkManager) ReadAt 

func (mcm *MinioChunkManager) ReadAt(ctx context.Context, bucketName string, filePath string, off int64, length int64) ([]byte, error)

ReadAt reads specific position data of minio storage if exists.

func (*MinioChunkManager) ReadWithPrefix 

func (mcm *MinioChunkManager) ReadWithPrefix(ctx context.Context, bucketName string, prefix string) ([]string, [][]byte, error)

func (*MinioChunkManager) Reader 

func (mcm *MinioChunkManager) Reader(ctx context.Context, bucketName string, filePath string) (FileReader, error)

Reader returns the path of minio data if exists.

func (*MinioChunkManager) Remove 

func (mcm *MinioChunkManager) Remove(ctx context.Context, bucketName string, filePath string) error

Remove deletes an object with @key.

func (*MinioChunkManager) RemoveWithPrefix 

func (mcm *MinioChunkManager) RemoveWithPrefix(ctx context.Context, bucketName string, prefix string) error

RemoveWithPrefix removes all objects with the same prefix @prefix from minio.

func (*MinioChunkManager) RootPath 

func (mcm *MinioChunkManager) RootPath() string

RootPath returns minio root path.

func (*MinioChunkManager) SetVar 

func (mcm *MinioChunkManager) SetVar(bucketName string, rootPath string)

SetVar set the variable value of mcm

func (*MinioChunkManager) Size 

func (mcm *MinioChunkManager) Size(ctx context.Context, bucketName string, filePath string) (int64, error)

func (*MinioChunkManager) Write 

func (mcm *MinioChunkManager) Write(ctx context.Context, bucketName string, filePath string, content []byte) error

Write writes the data to minio storage.

type Option 

type Option func(*config)

Option is used to config the retry function.

func AccessKeyID 

func AccessKeyID(accessKeyID string) Option

func Address 

func Address(addr string) Option

func BucketName 

func BucketName(bucketName string) Option

func CreateBucket 

func CreateBucket(createBucket bool) Option

func IAMEndpoint 

func IAMEndpoint(iamEndpoint string) Option

func RootPath 

func RootPath(rootPath string) Option

func SecretAccessKeyID 

func SecretAccessKeyID(secretAccessKeyID string) Option

func UseIAM 

func UseIAM(useIAM bool) Option

func UseSSL 

func UseSSL(useSSL bool) Option

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.