storage

package
v2.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 16, 2024 License: MIT Imports: 40 Imported by: 8

Documentation

Overview

Package storage is a generated GoMock package.

Package storage implements operations for s3 and fs.

Index

Constants

This section is empty.

Variables

View Source 
var ErrNoObjectFound = fmt.Errorf("no object found")

ErrNoObjectFound indicates there are no objects found from a given directory.

Functions

func FromBytes 

func FromBytes(data []byte) extsort.SortType

func IsCancelationError 

func IsCancelationError(err error) bool

IsCancelationError reports whether given error is a storage related cancelation error.

func IsGoogleEndpoint 

func IsGoogleEndpoint(endpoint urlpkg.URL) bool

func Less 

func Less(a, b extsort.SortType) bool

Less returns if relative path of storage.Object a's URL comes before the one of b's in the lexicographic order. It assumes that both a, and b are the instances of Object

func ShouldProcessURL 

func ShouldProcessURL(url *url.URL, followSymlinks bool) bool

ShouldProcessURL returns true if follow symlinks is enabled. If follow symlinks is disabled we should not process the url. (this check is needed only for local files)

Types

type Bucket 

type Bucket struct {
	CreationDate time.Time `json:"created_at"`
	Name         string    `json:"name"`
}

Bucket is a container for storage objects.

func (Bucket) JSON 

func (b Bucket) JSON() string

JSON returns the JSON representation of Bucket.

func (Bucket) String 

func (b Bucket) String() string

String returns the string representation of Bucket.

type CsvDecoder 

type CsvDecoder struct {
	// contains filtered or unexported fields
}

func (*CsvDecoder) Decode 

func (cd *CsvDecoder) Decode() ([]byte, error)

type ErrGivenObjectNotFound 

type ErrGivenObjectNotFound struct {
	ObjectAbsPath string
}

ErrGivenObjectNotFound indicates a specified object is not found.

func (*ErrGivenObjectNotFound) Error 

func (e *ErrGivenObjectNotFound) Error() string

type EventStreamDecoder 

type EventStreamDecoder interface {
	Decode() ([]byte, error)
}

EventStreamDecoder decodes a s3.Event with the given decoder.

func NewCsvDecoder 

func NewCsvDecoder(reader io.Reader) EventStreamDecoder

func NewJSONDecoder 

func NewJSONDecoder(reader io.Reader) EventStreamDecoder

type Filesystem 

type Filesystem struct {
	// contains filtered or unexported fields
}

Filesystem is the Storage implementation of a local filesystem.

func NewLocalClient 

func NewLocalClient(opts Options) *Filesystem

func (*Filesystem) Copy 

func (f *Filesystem) Copy(ctx context.Context, src, dst *url.URL, _ Metadata) error

Copy copies given source to destination.

func (*Filesystem) Create 

func (f *Filesystem) Create(path string) (*os.File, error)

Create creates a new os.File.

func (*Filesystem) CreateTemp 

func (f *Filesystem) CreateTemp(dir, pattern string) (*os.File, error)

CreateTemp creates a new temporary file

func (*Filesystem) Delete 

func (f *Filesystem) Delete(ctx context.Context, url *url.URL) error

Delete deletes given file.

func (*Filesystem) List 

func (f *Filesystem) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

List returns the objects and directories reside in given src.

func (*Filesystem) MkdirAll 

func (f *Filesystem) MkdirAll(path string) error

MkdirAll calls os.MkdirAll.

func (*Filesystem) MultiDelete 

func (f *Filesystem) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

MultiDelete deletes all files returned from given channel.

func (*Filesystem) Open 

func (f *Filesystem) Open(path string) (*os.File, error)

Open opens the given source.

func (*Filesystem) Rename 

func (f *Filesystem) Rename(file *os.File, newpath string) error

Rename a file

func (*Filesystem) Stat 

func (f *Filesystem) Stat(ctx context.Context, url *url.URL) (*Object, error)

Stat returns the Object structure describing object.

type JSONDecoder 

type JSONDecoder struct {
	// contains filtered or unexported fields
}

func (*JSONDecoder) Decode 

func (jd *JSONDecoder) Decode() ([]byte, error)

type Metadata 

type Metadata struct {
	ACL                string
	CacheControl       string
	Expires            string
	StorageClass       string
	ContentType        string
	ContentEncoding    string
	ContentDisposition string
	EncryptionMethod   string
	EncryptionKeyID    string

	UserDefined map[string]string

	// MetadataDirective is used to specify whether the metadata is copied from
	// the source object or replaced with metadata provided when copying S3
	// objects. If MetadataDirective is not set, it defaults to "COPY".
	Directive string
}

type MockStorage 

type MockStorage struct {
	// contains filtered or unexported fields
}

MockStorage is a mock of Storage interface.

func NewMockStorage 

func NewMockStorage(ctrl *gomock.Controller) *MockStorage

NewMockStorage creates a new mock instance.

func (*MockStorage) Copy 

func (m *MockStorage) Copy(ctx context.Context, src, dst *url.URL, metadata Metadata) error

Copy mocks base method.

func (*MockStorage) Delete 

func (m *MockStorage) Delete(ctx context.Context, src *url.URL) error

Delete mocks base method.

func (*MockStorage) EXPECT 

func (m *MockStorage) EXPECT() *MockStorageMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockStorage) List 

func (m *MockStorage) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

List mocks base method.

func (*MockStorage) MultiDelete 

func (m *MockStorage) MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

MultiDelete mocks base method.

func (*MockStorage) Stat 

func (m *MockStorage) Stat(ctx context.Context, src *url.URL) (*Object, error)

Stat mocks base method.

type MockStorageMockRecorder 

type MockStorageMockRecorder struct {
	// contains filtered or unexported fields
}

MockStorageMockRecorder is the mock recorder for MockStorage.

func (*MockStorageMockRecorder) Copy 

func (mr *MockStorageMockRecorder) Copy(ctx, src, dst, metadata any) *gomock.Call

Copy indicates an expected call of Copy.

func (*MockStorageMockRecorder) Delete 

func (mr *MockStorageMockRecorder) Delete(ctx, src any) *gomock.Call

Delete indicates an expected call of Delete.

func (*MockStorageMockRecorder) List 

func (mr *MockStorageMockRecorder) List(ctx, src, followSymlinks any) *gomock.Call

List indicates an expected call of List.

func (*MockStorageMockRecorder) MultiDelete 

func (mr *MockStorageMockRecorder) MultiDelete(ctx, urls any) *gomock.Call

MultiDelete indicates an expected call of MultiDelete.

func (*MockStorageMockRecorder) Stat 

func (mr *MockStorageMockRecorder) Stat(ctx, src any) *gomock.Call

Stat indicates an expected call of Stat.

type Object 

type Object struct {
	URL          *url.URL     `json:"key,omitempty"`
	Etag         string       `json:"etag,omitempty"`
	ModTime      *time.Time   `json:"last_modified,omitempty"`
	Type         ObjectType   `json:"type,omitempty"`
	Size         int64        `json:"size,omitempty"`
	StorageClass StorageClass `json:"storage_class,omitempty"`
	Err          error        `json:"error,omitempty"`

	// the VersionID field exist only for JSON Marshall, it must not be used for
	// any other purpose. URL.VersionID must be used instead.
	VersionID string `json:"version_id,omitempty"`
	// contains filtered or unexported fields
}

Object is a generic type which contains metadata for storage items.

func (*Object) JSON 

func (o *Object) JSON() string

JSON returns the JSON representation of Object.

func (*Object) String 

func (o *Object) String() string

String returns the string representation of Object.

func (Object) ToBytes 

func (o Object) ToBytes() []byte

type ObjectType 

type ObjectType struct {
	// contains filtered or unexported fields
}

ObjectType is the type of Object.

func (ObjectType) IsDir 

func (o ObjectType) IsDir() bool

IsDir checks if the object is a directory.

func (ObjectType) IsRegular 

func (o ObjectType) IsRegular() bool

IsRegular checks if the object is a regular file. 

func (o ObjectType) IsSymlink() bool

IsSymlink checks if the object is a symbolic link.

func (ObjectType) MarshalJSON 

func (o ObjectType) MarshalJSON() ([]byte, error)

MarshalJSON returns the stringer of ObjectType as a marshalled json.

func (ObjectType) String 

func (o ObjectType) String() string

String returns the string representation of ObjectType.

type Options 

type Options struct {
	MaxRetries             int
	NoSuchUploadRetryCount int
	Endpoint               string
	NoVerifySSL            bool
	DryRun                 bool
	NoSignRequest          bool
	UseListObjectsV1       bool
	LogLevel               log.LogLevel
	RequestPayer           string
	Profile                string
	CredentialFile         string
	// contains filtered or unexported fields
}

Options stores configuration for storage.

func (*Options) SetRegion 

func (o *Options) SetRegion(region string)

type S3 

type S3 struct {
	// contains filtered or unexported fields
}

S3 is a storage type which interacts with S3API, DownloaderAPI and UploaderAPI.

func NewRemoteClient 

func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error)

func (*S3) Copy 

func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) error

Copy is a single-object copy operation which copies objects to S3 destination from another S3 source.

func (*S3) Delete 

func (s *S3) Delete(ctx context.Context, url *url.URL) error

Delete is a single object delete operation.

func (*S3) Get 

func (s *S3) Get(
	ctx context.Context,
	from *url.URL,
	to io.WriterAt,
	concurrency int,
	partSize int64,
) (int64, error)

Get is a multipart download operation which downloads S3 objects into any destination that implements io.WriterAt interface. Makes a single 'GetObject' call if 'concurrency' is 1 and ignores 'partSize'.

func (*S3) GetBucketVersioning 

func (s *S3) GetBucketVersioning(ctx context.Context, bucket string) (string, error)

GetBucketVersioning returnsversioning property of the bucket

func (*S3) HeadBucket added in v2.3.0 

func (s *S3) HeadBucket(ctx context.Context, url *url.URL) error

func (*S3) HeadObject added in v2.3.0 

func (s *S3) HeadObject(ctx context.Context, url *url.URL) (*Object, *Metadata, error)

func (*S3) List 

func (s *S3) List(ctx context.Context, url *url.URL, _ bool) <-chan *Object

List is a non-blocking S3 list operation which paginates and filters S3 keys. If no object found or an error is encountered during this period, it sends these errors to object channel.

func (*S3) ListBuckets 

func (s *S3) ListBuckets(ctx context.Context, prefix string) ([]Bucket, error)

ListBuckets is a blocking list-operation which gets bucket list and returns the buckets that match with given prefix.

func (*S3) MakeBucket 

func (s *S3) MakeBucket(ctx context.Context, name string) error

MakeBucket creates an S3 bucket with the given name.

func (*S3) MultiDelete 

func (s *S3) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

MultiDelete is a asynchronous removal operation for multiple objects. It reads given url channel, creates multiple chunks and run these chunks in parallel. Each chunk may have at most 1000 objects since DeleteObjects API has a limitation. See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html.

func (*S3) Presign 

func (s *S3) Presign(ctx context.Context, from *url.URL, expire time.Duration) (string, error)

func (*S3) Put 

func (s *S3) Put(
	ctx context.Context,
	reader io.Reader,
	to *url.URL,
	metadata Metadata,
	concurrency int,
	partSize int64,
) error

Put is a multipart upload operation to upload resources, which implements io.Reader interface, into S3 destination.

func (*S3) Read 

func (s *S3) Read(ctx context.Context, src *url.URL) (io.ReadCloser, error)

Read fetches the remote object and returns its contents as an io.ReadCloser.

func (*S3) RemoveBucket 

func (s *S3) RemoveBucket(ctx context.Context, name string) error

RemoveBucket removes an S3 bucket with the given name.

func (*S3) RequestPayer 

func (s *S3) RequestPayer() *string

func (*S3) Select 

func (s *S3) Select(ctx context.Context, url *url.URL, query *SelectQuery, resultCh chan<- json.RawMessage) error

func (*S3) SetBucketVersioning 

func (s *S3) SetBucketVersioning(ctx context.Context, versioningStatus, bucket string) error

SetBucketVersioning sets the versioning property of the bucket

func (*S3) Stat 

func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error)

Stat retrieves metadata from S3 object without returning the object itself.

type SelectQuery 

type SelectQuery struct {
	InputFormat           string
	InputContentStructure string
	FileHeaderInfo        string
	OutputFormat          string
	ExpressionType        string
	Expression            string
	CompressionType       string
}

type SessionCache 

type SessionCache struct {
	sync.Mutex
	// contains filtered or unexported fields
}

SessionCache holds session.Session according to s3Opts and it synchronizes access/modification.

type Storage 

type Storage interface {
	// Stat returns the Object structure describing object. If src is not
	// found, ErrGivenObjectNotFound is returned.
	Stat(ctx context.Context, src *url.URL) (*Object, error)

	// List the objects and directories/prefixes in the src.
	List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

	// Delete deletes the given src.
	Delete(ctx context.Context, src *url.URL) error

	// MultiDelete deletes all items returned from given urls in batches.
	MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

	// Copy src to dst, optionally setting the given metadata. Src and dst
	// arguments are of the same type. If src is a remote type, server side
	// copying will be used.
	Copy(ctx context.Context, src, dst *url.URL, metadata Metadata) error
}

Storage is an interface for storage operations that is common to local filesystem and remote object storage.

func NewClient 

func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error)

type StorageClass 

type StorageClass string

StorageClass represents the storage used to store an object.

func (StorageClass) IsGlacier 

func (s StorageClass) IsGlacier() bool

Source Files

View all Source files

Directories

Path Synopsis
Package url abstracts local and remote file URLs.
 Package url abstracts local and remote file URLs.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.