s3

package
v2.1.0-rc2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 26, 2020 License: AGPL-3.0 Imports: 29 Imported by: 0

Documentation

Overview

Package s3 provides an endpoint for connecting to Amazon S3 or an S3-compatible storage

Index

Constants

View Source
const MaxCopyObjectSize = 1024 * 1024 * 1024 * 5

Max CopyObjectSize is 5GB => if greater, we have to switch to multipart

Variables

View Source
var (
	UserAgentAppName = "pydio.sync.client.s3"
	UserAgentVersion = "1.0"
)

Functions

func CopyObjectMultipart

func CopyObjectMultipart(ctx context.Context, client *minio.Core, srcObject minio.ObjectInfo, srcBucket, srcPath, destBucket, destPath string, metadata map[string]string, progress io.Reader) error

Types

type ChecksumMapper

type ChecksumMapper interface {
	// Get finds a checksum for a given eTag, returns false if not found
	Get(eTag string) (string, bool)
	// Set stores an eTag checksum couple if it does not already exists
	Set(eTag, checksum string)
	// Purge removes unknown values based on the full list of know values
	Purge(knownETags []string) int
}

ChecksumMapper maintains a mapping of eTags => checksum

type Client

type Client struct {
	Mc                          MockableMinio
	Bucket                      string
	RootPath                    string
	Host                        string
	ServerRequiresNormalization bool
	// contains filtered or unexported fields
}

Client wraps a Minio Client to speak with an S3-compatible backend

func NewClient

func NewClient(ctx context.Context, host string, key string, secret string, bucket string, rootPath string, secure bool, options model.EndpointOptions) (*Client, error)

func NewS3Mock

func NewS3Mock(bucketName ...string) *Client

func (*Client) ComputeChecksum

func (c *Client) ComputeChecksum(node *tree.Node) error

func (*Client) CreateNode

func (c *Client) CreateNode(ctx context.Context, node *tree.Node, updateIfExists bool) (err error)

func (*Client) DeleteNode

func (c *Client) DeleteNode(ctx context.Context, path string) (err error)

func (*Client) GetEndpointInfo

func (c *Client) GetEndpointInfo() model.EndpointInfo

func (*Client) GetReaderOn

func (c *Client) GetReaderOn(path string) (out io.ReadCloser, err error)

func (*Client) GetWriterOn

func (c *Client) GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)

func (*Client) LoadNode

func (c *Client) LoadNode(ctx context.Context, path string, extendedStats ...bool) (node *tree.Node, err error)

func (*Client) MoveNode

func (c *Client) MoveNode(ctx context.Context, oldPath string, newPath string) (err error)

func (*Client) SetChecksumMapper

func (c *Client) SetChecksumMapper(mapper ChecksumMapper, purgeAfterWalk bool)

SetChecksumMapper passes a ChecksumMapper storage that will prevent in-place copy of objects or metadata modification and store md5 for a given eTag locally instead.

func (*Client) SetPlainSizeComputer

func (c *Client) SetPlainSizeComputer(computer func(nodeUUID string) (int64, error))

SetPlainSizeComputer passes a computer function to extract plain size from an encrypted node

func (*Client) SetServerRequiresNormalization

func (c *Client) SetServerRequiresNormalization()

SetServerRequiresNormalization is used on MacOS to normalize UTF-8 chars to/from NFC/NFD

func (*Client) SkipRecomputeEtagByCopy

func (c *Client) SkipRecomputeEtagByCopy()

SkipRecomputeEtagByCopy sets a special behavior to avoir recomputing etags by in-place copying objects on storages that do not support this feature

func (*Client) Stat

func (c *Client) Stat(path string) (i os.FileInfo, err error)

func (*Client) UpdateNodeUuid

func (c *Client) UpdateNodeUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)

UpdateNodeUuid makes this endpoint an UuidReceiver

func (*Client) Walk

func (c *Client) Walk(walknFc model.WalkNodesFunc, root string, recursive bool) (err error)

func (*Client) Watch

func (c *Client) Watch(recursivePath string) (*model.WatchObject, error)

type MemChecksumMapper

type MemChecksumMapper struct {
	// contains filtered or unexported fields
}

MemChecksumMapper is an in-memory implementation for ChecksumMapper interface

func NewMemChecksumMapper

func NewMemChecksumMapper() *MemChecksumMapper

NewMemChecksumMapper instantiates a new ChecksumMapper

func (*MemChecksumMapper) Get

func (m *MemChecksumMapper) Get(eTag string) (string, bool)

Get finds a checksum by eTag

func (*MemChecksumMapper) Purge

func (m *MemChecksumMapper) Purge(knownETags []string) int

Purge compares existing eTags to stored eTags and removes unnecessary ones

func (*MemChecksumMapper) Set

func (m *MemChecksumMapper) Set(eTag, checksum string)

Set stores a checksum for a given eTag

type MinioClientMock

type MinioClientMock struct {
	// contains filtered or unexported fields
}

func (*MinioClientMock) BucketExists

func (c *MinioClientMock) BucketExists(string) (bool, error)

func (*MinioClientMock) CopyObject

func (c *MinioClientMock) CopyObject(dest minio.DestinationInfo, source minio.SourceInfo) error

func (*MinioClientMock) GetBucketTagging

func (c *MinioClientMock) GetBucketTagging(bucketName string) ([]minio.Tag, error)

func (*MinioClientMock) GetObject

func (c *MinioClientMock) GetObject(bucket string, path string, opts minio.GetObjectOptions) (object *minio.Object, err error)

func (*MinioClientMock) ListBuckets

func (c *MinioClientMock) ListBuckets() (bb []minio.BucketInfo, e error)

func (*MinioClientMock) ListBucketsWithContext

func (c *MinioClientMock) ListBucketsWithContext(ctx context.Context) (bb []minio.BucketInfo, e error)

func (*MinioClientMock) ListObjectsV2

func (c *MinioClientMock) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan minio.ObjectInfo

func (*MinioClientMock) ListenBucketNotification

func (c *MinioClientMock) ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan minio.NotificationInfo

func (*MinioClientMock) PutObject

func (c *MinioClientMock) PutObject(bucket string, path string, reader io.Reader, size int64, opts minio.PutObjectOptions) (n int64, err error)

func (*MinioClientMock) RemoveObject

func (c *MinioClientMock) RemoveObject(bucket string, path string) error

func (*MinioClientMock) StatObject

func (c *MinioClientMock) StatObject(bucket string, path string, opts minio.StatObjectOptions) (minio.ObjectInfo, error)

type MockableMinio

type MockableMinio interface {
	StatObject(bucket string, path string, opts minio.StatObjectOptions) (minio.ObjectInfo, error)
	RemoveObject(bucket string, path string) error
	PutObject(bucket string, path string, reader io.Reader, size int64, opts minio.PutObjectOptions) (n int64, err error)
	GetObject(bucket string, path string, opts minio.GetObjectOptions) (object *minio.Object, err error)
	ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan minio.ObjectInfo
	CopyObject(dest minio.DestinationInfo, source minio.SourceInfo) error
	ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan minio.NotificationInfo
	ListBuckets() ([]minio.BucketInfo, error)
	ListBucketsWithContext(ctx context.Context) ([]minio.BucketInfo, error)
	BucketExists(string) (bool, error)
	GetBucketTagging(string) ([]minio.Tag, error)
}

type MultiBucketClient

type MultiBucketClient struct {
	// contains filtered or unexported fields
}

func NewMultiBucketClient

func NewMultiBucketClient(ctx context.Context, host string, key string, secret string, secure bool, options model.EndpointOptions, bucketsFilter string) (*MultiBucketClient, error)

NewMultiBucketClient creates an s3 wrapped client that lists buckets as top level folders

func (*MultiBucketClient) ComputeChecksum

func (m *MultiBucketClient) ComputeChecksum(node *tree.Node) (err error)

func (*MultiBucketClient) CreateNode

func (m *MultiBucketClient) CreateNode(ctx context.Context, node *tree.Node, updateIfExists bool) (err error)

func (*MultiBucketClient) DeleteNode

func (m *MultiBucketClient) DeleteNode(ctx context.Context, path string) (err error)

func (*MultiBucketClient) GetEndpointInfo

func (m *MultiBucketClient) GetEndpointInfo() model.EndpointInfo

func (*MultiBucketClient) GetReaderOn

func (m *MultiBucketClient) GetReaderOn(path string) (out io.ReadCloser, err error)

func (*MultiBucketClient) GetWriterOn

func (m *MultiBucketClient) GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)

func (*MultiBucketClient) LoadNode

func (m *MultiBucketClient) LoadNode(ctx context.Context, p string, extendedStats ...bool) (node *tree.Node, err error)

func (*MultiBucketClient) MoveNode

func (m *MultiBucketClient) MoveNode(ctx context.Context, oldPath string, newPath string) (err error)

func (*MultiBucketClient) ProvidesMetadataNamespaces

func (m *MultiBucketClient) ProvidesMetadataNamespaces() (out []glob.Glob, o bool)

ProvidesMetadataNamespaces implements MetadataProvider interface

func (*MultiBucketClient) SetChecksumMapper

func (m *MultiBucketClient) SetChecksumMapper(cs ChecksumMapper)

func (*MultiBucketClient) SetPlainSizeComputer

func (m *MultiBucketClient) SetPlainSizeComputer(computer func(nodeUUID string) (int64, error))

func (*MultiBucketClient) SetServerRequiresNormalization

func (m *MultiBucketClient) SetServerRequiresNormalization()

func (*MultiBucketClient) SkipRecomputeEtagByCopy

func (m *MultiBucketClient) SkipRecomputeEtagByCopy()

SkipRecomputeEtagByCopy sets a special behavior to avoir recomputing etags by in-place copying objects on storages that do not support this feature

func (*MultiBucketClient) UpdateNodeUuid

func (m *MultiBucketClient) UpdateNodeUuid(ctx context.Context, node *tree.Node) (n *tree.Node, err error)

func (*MultiBucketClient) Walk

func (m *MultiBucketClient) Walk(walknFc model.WalkNodesFunc, root string, recursive bool) (err error)

func (*MultiBucketClient) Watch

func (m *MultiBucketClient) Watch(recursivePath string) (*model.WatchObject, error)

type S3FileInfo

type S3FileInfo struct {
	Object minio.ObjectInfo
	// contains filtered or unexported fields
}

func NewS3FileInfo

func NewS3FileInfo(object minio.ObjectInfo) *S3FileInfo

func NewS3FolderInfo

func NewS3FolderInfo(object minio.ObjectInfo) *S3FileInfo

func (*S3FileInfo) IsDir

func (s *S3FileInfo) IsDir() bool

abbreviation for Mode().IsDir()

func (*S3FileInfo) ModTime

func (s *S3FileInfo) ModTime() time.Time

modification time

func (*S3FileInfo) Mode

func (s *S3FileInfo) Mode() os.FileMode

file mode bits

func (*S3FileInfo) Name

func (s *S3FileInfo) Name() string

base name of the file

func (*S3FileInfo) Size

func (s *S3FileInfo) Size() int64

length in bytes for regular files; system-dependent for others

func (*S3FileInfo) Sys

func (s *S3FileInfo) Sys() interface{}

underlying data source (can return nil)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL