s3

package
v2.1.0-rc2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 26, 2020 License: AGPL-3.0 Imports: 29 Imported by: 0

Documentation

Overview

Package s3 provides an endpoint for connecting to Amazon S3 or an S3-compatible storage

Index

Constants

View Source 
const MaxCopyObjectSize = 1024 * 1024 * 1024 * 5

Max CopyObjectSize is 5GB => if greater, we have to switch to multipart

Variables

View Source 
var (
	UserAgentAppName = "pydio.sync.client.s3"
	UserAgentVersion = "1.0"
)

Functions

func CopyObjectMultipart 

func CopyObjectMultipart(ctx context.Context, client *minio.Core, srcObject minio.ObjectInfo, srcBucket, srcPath, destBucket, destPath string, metadata map[string]string, progress io.Reader) error

Types

type ChecksumMapper 

type ChecksumMapper interface {
	// Get finds a checksum for a given eTag, returns false if not found
	Get(eTag string) (string, bool)
	// Set stores an eTag checksum couple if it does not already exists
	Set(eTag, checksum string)
	// Purge removes unknown values based on the full list of know values
	Purge(knownETags []string) int
}

ChecksumMapper maintains a mapping of eTags => checksum

type Client 

type Client struct {
	Mc                          MockableMinio
	Bucket                      string
	RootPath                    string
	Host                        string
	ServerRequiresNormalization bool
	// contains filtered or unexported fields
}

Client wraps a Minio Client to speak with an S3-compatible backend

func NewClient 

func NewClient(ctx context.Context, host string, key string, secret string, bucket string, rootPath string, secure bool, options model.EndpointOptions) (*Client, error)

func NewS3Mock 

func NewS3Mock(bucketName ...string) *Client

func (*Client) ComputeChecksum 

func (c *Client) ComputeChecksum(node *tree.Node) error

func (*Client) CreateNode 

func (c *Client) CreateNode(ctx context.Context, node *tree.Node, updateIfExists bool) (err error)

func (*Client) DeleteNode 

func (c *Client) DeleteNode(ctx context.Context, path string) (err error)

func (*Client) GetEndpointInfo 

func (c *Client) GetEndpointInfo() model.EndpointInfo

func (*Client) GetReaderOn 

func (c *Client) GetReaderOn(path string) (out io.ReadCloser, err error)

func (*Client) GetWriterOn 

func (c *Client) GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)

func (*Client) LoadNode 

func (c *Client) LoadNode(ctx context.Context, path string, extendedStats ...bool) (node *tree.Node, err error)

func (*Client) MoveNode 

func (c *Client) MoveNode(ctx context.Context, oldPath string, newPath string) (err error)

func (*Client) SetChecksumMapper 

func (c *Client) SetChecksumMapper(mapper ChecksumMapper, purgeAfterWalk bool)

SetChecksumMapper passes a ChecksumMapper storage that will prevent in-place copy of objects or metadata modification and store md5 for a given eTag locally instead.

func (*Client) SetPlainSizeComputer 

func (c *Client) SetPlainSizeComputer(computer func(nodeUUID string) (int64, error))

SetPlainSizeComputer passes a computer function to extract plain size from an encrypted node

func (*Client) SetServerRequiresNormalization 

func (c *Client) SetServerRequiresNormalization()

SetServerRequiresNormalization is used on MacOS to normalize UTF-8 chars to/from NFC/NFD

func (*Client) SkipRecomputeEtagByCopy 

func (c *Client) SkipRecomputeEtagByCopy()

SkipRecomputeEtagByCopy sets a special behavior to avoir recomputing etags by in-place copying objects on storages that do not support this feature

func (*Client) Stat 

func (c *Client) Stat(path string) (i os.FileInfo, err error)

func (*Client) UpdateNodeUuid 

func (c *Client) UpdateNodeUuid(ctx context.Context, node *tree.Node) (*tree.Node, error)

UpdateNodeUuid makes this endpoint an UuidReceiver

func (*Client) Walk 

func (c *Client) Walk(walknFc model.WalkNodesFunc, root string, recursive bool) (err error)

func (*Client) Watch 

func (c *Client) Watch(recursivePath string) (*model.WatchObject, error)

type MemChecksumMapper 

type MemChecksumMapper struct {
	// contains filtered or unexported fields
}

MemChecksumMapper is an in-memory implementation for ChecksumMapper interface

func NewMemChecksumMapper 

func NewMemChecksumMapper() *MemChecksumMapper

NewMemChecksumMapper instantiates a new ChecksumMapper

func (*MemChecksumMapper) Get 

func (m *MemChecksumMapper) Get(eTag string) (string, bool)

Get finds a checksum by eTag

func (*MemChecksumMapper) Purge 

func (m *MemChecksumMapper) Purge(knownETags []string) int

Purge compares existing eTags to stored eTags and removes unnecessary ones

func (*MemChecksumMapper) Set 

func (m *MemChecksumMapper) Set(eTag, checksum string)

Set stores a checksum for a given eTag

type MinioClientMock 

type MinioClientMock struct {
	// contains filtered or unexported fields
}

func (*MinioClientMock) BucketExists 

func (c *MinioClientMock) BucketExists(string) (bool, error)

func (*MinioClientMock) CopyObject 

func (c *MinioClientMock) CopyObject(dest minio.DestinationInfo, source minio.SourceInfo) error

func (*MinioClientMock) GetBucketTagging 

func (c *MinioClientMock) GetBucketTagging(bucketName string) ([]minio.Tag, error)

func (*MinioClientMock) GetObject 

func (c *MinioClientMock) GetObject(bucket string, path string, opts minio.GetObjectOptions) (object *minio.Object, err error)

func (*MinioClientMock) ListBuckets 

func (c *MinioClientMock) ListBuckets() (bb []minio.BucketInfo, e error)

func (*MinioClientMock) ListBucketsWithContext 

func (c *MinioClientMock) ListBucketsWithContext(ctx context.Context) (bb []minio.BucketInfo, e error)

func (*MinioClientMock) ListObjectsV2 

func (c *MinioClientMock) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan minio.ObjectInfo

func (*MinioClientMock) ListenBucketNotification 

func (c *MinioClientMock) ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan minio.NotificationInfo

func (*MinioClientMock) PutObject 

func (c *MinioClientMock) PutObject(bucket string, path string, reader io.Reader, size int64, opts minio.PutObjectOptions) (n int64, err error)

func (*MinioClientMock) RemoveObject 

func (c *MinioClientMock) RemoveObject(bucket string, path string) error

func (*MinioClientMock) StatObject 

func (c *MinioClientMock) StatObject(bucket string, path string, opts minio.StatObjectOptions) (minio.ObjectInfo, error)

type MockableMinio 

type MockableMinio interface {
	StatObject(bucket string, path string, opts minio.StatObjectOptions) (minio.ObjectInfo, error)
	RemoveObject(bucket string, path string) error
	PutObject(bucket string, path string, reader io.Reader, size int64, opts minio.PutObjectOptions) (n int64, err error)
	GetObject(bucket string, path string, opts minio.GetObjectOptions) (object *minio.Object, err error)
	ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan minio.ObjectInfo
	CopyObject(dest minio.DestinationInfo, source minio.SourceInfo) error
	ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan minio.NotificationInfo
	ListBuckets() ([]minio.BucketInfo, error)
	ListBucketsWithContext(ctx context.Context) ([]minio.BucketInfo, error)
	BucketExists(string) (bool, error)
	GetBucketTagging(string) ([]minio.Tag, error)
}

type MultiBucketClient 

type MultiBucketClient struct {
	// contains filtered or unexported fields
}

func NewMultiBucketClient 

func NewMultiBucketClient(ctx context.Context, host string, key string, secret string, secure bool, options model.EndpointOptions, bucketsFilter string) (*MultiBucketClient, error)

NewMultiBucketClient creates an s3 wrapped client that lists buckets as top level folders

func (*MultiBucketClient) ComputeChecksum 

func (m *MultiBucketClient) ComputeChecksum(node *tree.Node) (err error)

func (*MultiBucketClient) CreateNode 

func (m *MultiBucketClient) CreateNode(ctx context.Context, node *tree.Node, updateIfExists bool) (err error)

func (*MultiBucketClient) DeleteNode 

func (m *MultiBucketClient) DeleteNode(ctx context.Context, path string) (err error)

func (*MultiBucketClient) GetEndpointInfo 

func (m *MultiBucketClient) GetEndpointInfo() model.EndpointInfo

func (*MultiBucketClient) GetReaderOn 

func (m *MultiBucketClient) GetReaderOn(path string) (out io.ReadCloser, err error)

func (*MultiBucketClient) GetWriterOn 

func (m *MultiBucketClient) GetWriterOn(cancel context.Context, path string, targetSize int64) (out io.WriteCloser, writeDone chan bool, writeErr chan error, err error)

func (*MultiBucketClient) LoadNode 

func (m *MultiBucketClient) LoadNode(ctx context.Context, p string, extendedStats ...bool) (node *tree.Node, err error)

func (*MultiBucketClient) MoveNode 

func (m *MultiBucketClient) MoveNode(ctx context.Context, oldPath string, newPath string) (err error)

func (*MultiBucketClient) ProvidesMetadataNamespaces 

func (m *MultiBucketClient) ProvidesMetadataNamespaces() (out []glob.Glob, o bool)

ProvidesMetadataNamespaces implements MetadataProvider interface

func (*MultiBucketClient) SetChecksumMapper 

func (m *MultiBucketClient) SetChecksumMapper(cs ChecksumMapper)

func (*MultiBucketClient) SetPlainSizeComputer 

func (m *MultiBucketClient) SetPlainSizeComputer(computer func(nodeUUID string) (int64, error))

func (*MultiBucketClient) SetServerRequiresNormalization 

func (m *MultiBucketClient) SetServerRequiresNormalization()

func (*MultiBucketClient) SkipRecomputeEtagByCopy 

func (m *MultiBucketClient) SkipRecomputeEtagByCopy()

SkipRecomputeEtagByCopy sets a special behavior to avoir recomputing etags by in-place copying objects on storages that do not support this feature

func (*MultiBucketClient) UpdateNodeUuid 

func (m *MultiBucketClient) UpdateNodeUuid(ctx context.Context, node *tree.Node) (n *tree.Node, err error)

func (*MultiBucketClient) Walk 

func (m *MultiBucketClient) Walk(walknFc model.WalkNodesFunc, root string, recursive bool) (err error)

func (*MultiBucketClient) Watch 

func (m *MultiBucketClient) Watch(recursivePath string) (*model.WatchObject, error)

type S3FileInfo 

type S3FileInfo struct {
	Object minio.ObjectInfo
	// contains filtered or unexported fields
}

func NewS3FileInfo 

func NewS3FileInfo(object minio.ObjectInfo) *S3FileInfo

func NewS3FolderInfo 

func NewS3FolderInfo(object minio.ObjectInfo) *S3FileInfo

func (*S3FileInfo) IsDir 

func (s *S3FileInfo) IsDir() bool

abbreviation for Mode().IsDir()

func (*S3FileInfo) ModTime 

func (s *S3FileInfo) ModTime() time.Time

modification time

func (*S3FileInfo) Mode 

func (s *S3FileInfo) Mode() os.FileMode

file mode bits

func (*S3FileInfo) Name 

func (s *S3FileInfo) Name() string

base name of the file

func (*S3FileInfo) Size 

func (s *S3FileInfo) Size() int64

length in bytes for regular files; system-dependent for others

func (*S3FileInfo) Sys 

func (s *S3FileInfo) Sys() interface{}

underlying data source (can return nil)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.