storage

package
v0.0.0-...-9c41f32 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 16, 2022 License: MIT Imports: 34 Imported by: 0

Documentation

Overview

Package storage implements operations for s3 and fs.

Index

Constants

This section is empty.

Variables

View Source 
var (

	// ErrNoObjectFound indicates there are no objects found from a given directory.
	ErrNoObjectFound = fmt.Errorf("no object found")
)

Functions

func IsCancelationError 

func IsCancelationError(err error) bool

IsCancelationError reports whether given error is a storage related cancelation error.

func IsGoogleEndpoint 

func IsGoogleEndpoint(endpoint urlpkg.URL) bool

func ShouldProcessUrl 

func ShouldProcessUrl(url *url.URL, followSymlinks bool) bool

ShouldProcessUrl returns true if follow symlinks is enabled. If follow symlinks is disabled we should not process the url. (this check is needed only for local files)

Types

type Bucket 

type Bucket struct {
	CreationDate time.Time `json:"created_at"`
	Name         string    `json:"name"`
}

Bucket is a container for storage objects.

func (Bucket) JSON 

func (b Bucket) JSON() string

JSON returns the JSON representation of Bucket.

func (Bucket) String 

func (b Bucket) String() string

String returns the string representation of Bucket.

type ErrGivenObjectNotFound 

type ErrGivenObjectNotFound struct {
	ObjectAbsPath string
}

ErrGivenObjectNotFound indicates a specified object is not found.

func (*ErrGivenObjectNotFound) Error 

func (e *ErrGivenObjectNotFound) Error() string

type Filesystem 

type Filesystem struct {
	// contains filtered or unexported fields
}

Filesystem is the Storage implementation of a local filesystem.

func NewLocalClient 

func NewLocalClient(opts Options) *Filesystem

func (*Filesystem) Copy 

func (f *Filesystem) Copy(ctx context.Context, src, dst *url.URL, _ Metadata) error

Copy copies given source to destination.

func (*Filesystem) Create 

func (f *Filesystem) Create(path string) (*os.File, error)

Create creates a new os.File.

func (*Filesystem) Delete 

func (f *Filesystem) Delete(ctx context.Context, url *url.URL) error

Delete deletes given file.

func (*Filesystem) List 

func (f *Filesystem) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

List returns the objects and directories reside in given src.

func (*Filesystem) MkdirAll 

func (f *Filesystem) MkdirAll(path string) error

MkdirAll calls os.MkdirAll.

func (*Filesystem) MultiDelete 

func (f *Filesystem) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

MultiDelete deletes all files returned from given channel.

func (*Filesystem) Open 

func (f *Filesystem) Open(path string) (*os.File, error)

Open opens the given source.

func (*Filesystem) Stat 

func (f *Filesystem) Stat(ctx context.Context, url *url.URL) (*Object, error)

Stat returns the Object structure describing object.

type Metadata 

type Metadata map[string]string

func NewMetadata 

func NewMetadata() Metadata

NewMetadata will return an empty metadata object.

func (Metadata) ACL 

func (m Metadata) ACL() string

func (Metadata) CacheControl 

func (m Metadata) CacheControl() string

func (Metadata) ContentEncoding 

func (m Metadata) ContentEncoding() string

func (Metadata) ContentType 

func (m Metadata) ContentType() string

func (Metadata) Expires 

func (m Metadata) Expires() string

func (Metadata) SSE 

func (m Metadata) SSE() string

func (Metadata) SSEKeyID 

func (m Metadata) SSEKeyID() string

func (Metadata) SetACL 

func (m Metadata) SetACL(acl string) Metadata

func (Metadata) SetCacheControl 

func (m Metadata) SetCacheControl(cacheControl string) Metadata

func (Metadata) SetContentEncoding 

func (m Metadata) SetContentEncoding(contentEncoding string) Metadata

func (Metadata) SetContentType 

func (m Metadata) SetContentType(contentType string) Metadata

func (Metadata) SetExpires 

func (m Metadata) SetExpires(expires string) Metadata

func (Metadata) SetSSE 

func (m Metadata) SetSSE(sse string) Metadata

func (Metadata) SetSSEKeyID 

func (m Metadata) SetSSEKeyID(kid string) Metadata

func (Metadata) SetStorageClass 

func (m Metadata) SetStorageClass(class string) Metadata

func (Metadata) StorageClass 

func (m Metadata) StorageClass() string

type MockStorage 

type MockStorage struct {
	mock.Mock
}

MockStorage is an autogenerated mock type for the Storage type

func (*MockStorage) Copy 

func (_m *MockStorage) Copy(ctx context.Context, src *url.URL, dst *url.URL, metadata Metadata) error

Copy provides a mock function with given fields: ctx, src, dst, metadata

func (*MockStorage) Delete 

func (_m *MockStorage) Delete(ctx context.Context, src *url.URL) error

Delete provides a mock function with given fields: ctx, src

func (*MockStorage) List 

func (_m *MockStorage) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

List provides a mock function with given fields: ctx, src, followSymlinks

func (*MockStorage) MultiDelete 

func (_m *MockStorage) MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

MultiDelete provides a mock function with given fields: ctx, urls

func (*MockStorage) Stat 

func (_m *MockStorage) Stat(ctx context.Context, src *url.URL) (*Object, error)

Stat provides a mock function with given fields: ctx, src

type Object 

type Object struct {
	URL          *url.URL     `json:"key,omitempty"`
	Etag         string       `json:"etag,omitempty"`
	ModTime      *time.Time   `json:"last_modified,omitempty"`
	Type         ObjectType   `json:"type,omitempty"`
	Size         int64        `json:"size,omitempty"`
	StorageClass StorageClass `json:"storage_class,omitempty"`
	Err          error        `json:"error,omitempty"`
	// contains filtered or unexported fields
}

Object is a generic type which contains metadata for storage items.

func (*Object) JSON 

func (o *Object) JSON() string

JSON returns the JSON representation of Object.

func (*Object) String 

func (o *Object) String() string

String returns the string representation of Object.

type ObjectType 

type ObjectType struct {
	// contains filtered or unexported fields
}

ObjectType is the type of Object.

func (ObjectType) IsDir 

func (o ObjectType) IsDir() bool

IsDir checks if the object is a directory. 

func (o ObjectType) IsSymlink() bool

IsSymlink checks if the object is a symbolic link.

func (ObjectType) MarshalJSON 

func (o ObjectType) MarshalJSON() ([]byte, error)

MarshalJSON returns the stringer of ObjectType as a marshalled json.

func (ObjectType) String 

func (o ObjectType) String() string

String returns the string representation of ObjectType.

type Options 

type Options struct {
	MaxRetries             int
	NoSuchUploadRetryCount int
	Endpoint               string
	NoVerifySSL            bool
	DryRun                 bool
	NoSignRequest          bool
	UseListObjectsV1       bool
	LogLevel               log.LogLevel
	RequestPayer           string
	Profile                string
	CredentialFile         string
	// contains filtered or unexported fields
}

Options stores configuration for storage.

func (*Options) SetRegion 

func (o *Options) SetRegion(region string)

type S3 

type S3 struct {
	// contains filtered or unexported fields
}

S3 is a storage type which interacts with S3API, DownloaderAPI and UploaderAPI.

func NewRemoteClient 

func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error)

func (*S3) Copy 

func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) error

Copy is a single-object copy operation which copies objects to S3 destination from another S3 source.

func (*S3) Delete 

func (s *S3) Delete(ctx context.Context, url *url.URL) error

Delete is a single object delete operation.

func (*S3) Get 

func (s *S3) Get(
	ctx context.Context,
	from *url.URL,
	to io.WriterAt,
	concurrency int,
	partSize int64,
) (int64, error)

Get is a multipart download operation which downloads S3 objects into any destination that implements io.WriterAt interface. Makes a single 'GetObject' call if 'concurrency' is 1 and ignores 'partSize'.

func (*S3) List 

func (s *S3) List(ctx context.Context, url *url.URL, _ bool) <-chan *Object

List is a non-blocking S3 list operation which paginates and filters S3 keys. If no object found or an error is encountered during this period, it sends these errors to object channel.

func (*S3) ListBuckets 

func (s *S3) ListBuckets(ctx context.Context, prefix string) ([]Bucket, error)

ListBuckets is a blocking list-operation which gets bucket list and returns the buckets that match with given prefix.

func (*S3) MakeBucket 

func (s *S3) MakeBucket(ctx context.Context, name string) error

MakeBucket creates an S3 bucket with the given name.

func (*S3) MultiDelete 

func (s *S3) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

MultiDelete is a asynchronous removal operation for multiple objects. It reads given url channel, creates multiple chunks and run these chunks in parallel. Each chunk may have at most 1000 objects since DeleteObjects API has a limitation. See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html.

func (*S3) Put 

func (s *S3) Put(
	ctx context.Context,
	reader io.Reader,
	to *url.URL,
	metadata Metadata,
	concurrency int,
	partSize int64,
) error

Put is a multipart upload operation to upload resources, which implements io.Reader interface, into S3 destination.

func (*S3) Read 

func (s *S3) Read(ctx context.Context, src *url.URL) (io.ReadCloser, error)

Read fetches the remote object and returns its contents as an io.ReadCloser.

func (*S3) RemoveBucket 

func (s *S3) RemoveBucket(ctx context.Context, name string) error

RemoveBucket removes an S3 bucket with the given name.

func (*S3) RequestPayer 

func (s *S3) RequestPayer() *string

func (*S3) Select 

func (s *S3) Select(ctx context.Context, url *url.URL, query *SelectQuery, resultCh chan<- json.RawMessage) error

func (*S3) Stat 

func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error)

Stat retrieves metadata from S3 object without returning the object itself.

type SelectQuery 

type SelectQuery struct {
	ExpressionType  string
	Expression      string
	CompressionType string
}

type SessionCache 

type SessionCache struct {
	sync.Mutex
	// contains filtered or unexported fields
}

SessionCache holds session.Session according to s3Opts and it synchronizes access/modification.

type Storage 

type Storage interface {
	// Stat returns the Object structure describing object. If src is not
	// found, ErrGivenObjectNotFound is returned.
	Stat(ctx context.Context, src *url.URL) (*Object, error)

	// List the objects and directories/prefixes in the src.
	List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

	// Delete deletes the given src.
	Delete(ctx context.Context, src *url.URL) error

	// MultiDelete deletes all items returned from given urls in batches.
	MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

	// Copy src to dst, optionally setting the given metadata. Src and dst
	// arguments are of the same type. If src is a remote type, server side
	// copying will be used.
	Copy(ctx context.Context, src, dst *url.URL, metadata Metadata) error
}

Storage is an interface for storage operations that is common to local filesystem and remote object storage.

func NewClient 

func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error)

type StorageClass 

type StorageClass string

StorageClass represents the storage used to store an object.

func (StorageClass) IsGlacier 

func (s StorageClass) IsGlacier() bool

Source Files

View all Source files

Directories

Path Synopsis
Package url abstracts local and remote file URLs.
 Package url abstracts local and remote file URLs.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.